{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# 逻辑回归案例 银行卡信用问题\n",
    "\n",
    "## 数据\n",
    "\n",
    "credit_card数据集(逻辑回归)\n",
    "\n",
    "数据集中包含284807行31列数据\n",
    "\n",
    "问题：银行卡信用问题，根据已有的30个的特征及class进行分类，判断为正常或异常情况\n",
    "\n",
    "数据资源:  https://pan.baidu.com/s/1fzqeieHOrBmV5TJ1RfhBbw 提取码: buiu  或 https://www.kaggle.com/jacklizhi/creditcard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 引入数据和模块\n",
    "import os\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Time</th>\n",
       "      <th>V1</th>\n",
       "      <th>V2</th>\n",
       "      <th>V3</th>\n",
       "      <th>V4</th>\n",
       "      <th>V5</th>\n",
       "      <th>V6</th>\n",
       "      <th>V7</th>\n",
       "      <th>V8</th>\n",
       "      <th>V9</th>\n",
       "      <th>...</th>\n",
       "      <th>V21</th>\n",
       "      <th>V22</th>\n",
       "      <th>V23</th>\n",
       "      <th>V24</th>\n",
       "      <th>V25</th>\n",
       "      <th>V26</th>\n",
       "      <th>V27</th>\n",
       "      <th>V28</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.359807</td>\n",
       "      <td>-0.072781</td>\n",
       "      <td>2.536347</td>\n",
       "      <td>1.378155</td>\n",
       "      <td>-0.338321</td>\n",
       "      <td>0.462388</td>\n",
       "      <td>0.239599</td>\n",
       "      <td>0.098698</td>\n",
       "      <td>0.363787</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.018307</td>\n",
       "      <td>0.277838</td>\n",
       "      <td>-0.110474</td>\n",
       "      <td>0.066928</td>\n",
       "      <td>0.128539</td>\n",
       "      <td>-0.189115</td>\n",
       "      <td>0.133558</td>\n",
       "      <td>-0.021053</td>\n",
       "      <td>149.62</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.191857</td>\n",
       "      <td>0.266151</td>\n",
       "      <td>0.166480</td>\n",
       "      <td>0.448154</td>\n",
       "      <td>0.060018</td>\n",
       "      <td>-0.082361</td>\n",
       "      <td>-0.078803</td>\n",
       "      <td>0.085102</td>\n",
       "      <td>-0.255425</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.225775</td>\n",
       "      <td>-0.638672</td>\n",
       "      <td>0.101288</td>\n",
       "      <td>-0.339846</td>\n",
       "      <td>0.167170</td>\n",
       "      <td>0.125895</td>\n",
       "      <td>-0.008983</td>\n",
       "      <td>0.014724</td>\n",
       "      <td>2.69</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.358354</td>\n",
       "      <td>-1.340163</td>\n",
       "      <td>1.773209</td>\n",
       "      <td>0.379780</td>\n",
       "      <td>-0.503198</td>\n",
       "      <td>1.800499</td>\n",
       "      <td>0.791461</td>\n",
       "      <td>0.247676</td>\n",
       "      <td>-1.514654</td>\n",
       "      <td>...</td>\n",
       "      <td>0.247998</td>\n",
       "      <td>0.771679</td>\n",
       "      <td>0.909412</td>\n",
       "      <td>-0.689281</td>\n",
       "      <td>-0.327642</td>\n",
       "      <td>-0.139097</td>\n",
       "      <td>-0.055353</td>\n",
       "      <td>-0.059752</td>\n",
       "      <td>378.66</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.966272</td>\n",
       "      <td>-0.185226</td>\n",
       "      <td>1.792993</td>\n",
       "      <td>-0.863291</td>\n",
       "      <td>-0.010309</td>\n",
       "      <td>1.247203</td>\n",
       "      <td>0.237609</td>\n",
       "      <td>0.377436</td>\n",
       "      <td>-1.387024</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.108300</td>\n",
       "      <td>0.005274</td>\n",
       "      <td>-0.190321</td>\n",
       "      <td>-1.175575</td>\n",
       "      <td>0.647376</td>\n",
       "      <td>-0.221929</td>\n",
       "      <td>0.062723</td>\n",
       "      <td>0.061458</td>\n",
       "      <td>123.50</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>-1.158233</td>\n",
       "      <td>0.877737</td>\n",
       "      <td>1.548718</td>\n",
       "      <td>0.403034</td>\n",
       "      <td>-0.407193</td>\n",
       "      <td>0.095921</td>\n",
       "      <td>0.592941</td>\n",
       "      <td>-0.270533</td>\n",
       "      <td>0.817739</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.009431</td>\n",
       "      <td>0.798278</td>\n",
       "      <td>-0.137458</td>\n",
       "      <td>0.141267</td>\n",
       "      <td>-0.206010</td>\n",
       "      <td>0.502292</td>\n",
       "      <td>0.219422</td>\n",
       "      <td>0.215153</td>\n",
       "      <td>69.99</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Time        V1        V2        V3        V4        V5        V6        V7  \\\n",
       "0   0.0 -1.359807 -0.072781  2.536347  1.378155 -0.338321  0.462388  0.239599   \n",
       "1   0.0  1.191857  0.266151  0.166480  0.448154  0.060018 -0.082361 -0.078803   \n",
       "2   1.0 -1.358354 -1.340163  1.773209  0.379780 -0.503198  1.800499  0.791461   \n",
       "3   1.0 -0.966272 -0.185226  1.792993 -0.863291 -0.010309  1.247203  0.237609   \n",
       "4   2.0 -1.158233  0.877737  1.548718  0.403034 -0.407193  0.095921  0.592941   \n",
       "\n",
       "         V8        V9  ...       V21       V22       V23       V24       V25  \\\n",
       "0  0.098698  0.363787  ... -0.018307  0.277838 -0.110474  0.066928  0.128539   \n",
       "1  0.085102 -0.255425  ... -0.225775 -0.638672  0.101288 -0.339846  0.167170   \n",
       "2  0.247676 -1.514654  ...  0.247998  0.771679  0.909412 -0.689281 -0.327642   \n",
       "3  0.377436 -1.387024  ... -0.108300  0.005274 -0.190321 -1.175575  0.647376   \n",
       "4 -0.270533  0.817739  ... -0.009431  0.798278 -0.137458  0.141267 -0.206010   \n",
       "\n",
       "        V26       V27       V28  Amount  Class  \n",
       "0 -0.189115  0.133558 -0.021053  149.62      0  \n",
       "1  0.125895 -0.008983  0.014724    2.69      0  \n",
       "2 -0.139097 -0.055353 -0.059752  378.66      0  \n",
       "3 -0.221929  0.062723  0.061458  123.50      0  \n",
       "4  0.502292  0.219422  0.215153   69.99      0  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('data/creditcard.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Class = 0 为 正\n",
    "1 为 异  二分类\n",
    "\n",
    "使用逻辑回归建模，再进行二分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1       492\n",
       "0    284315\n",
       "Name: Class, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_class = pd.value_counts(data['Class'], sort=True).sort_values()\n",
    "count_class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Frequency')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAETCAYAAAD6R0vDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAYRElEQVR4nO3dfbRddX3n8ffHAAo+8CBpxAQI1qhFVAoRmbG2WpYQcFpwRi22lcgwREcYtXVNRZdrcKmspWtVqVhBoaaAVRClaiooIj6NMwshKAMEpMQAkhghkPCsPH7nj/O7cgg39x5gn3O9975fa5119/nu3977e7Ky8sne+3f3SVUhSVKXnjLVDUiSZh7DRZLUOcNFktQ5w0WS1DnDRZLUOcNFktQ5w0Vqkpye5CNT3ceYJLsluTvJnKnuRXq8DBfNGul5Z5KrktyTZG2SLyd5yRT0sjBJJdlqs/pvA66qflFVz6iqhybZ11uT/GiY/UqPl+Gi2eSTwLuAdwI7AS8Avga8bgp7+p3WAtl/J/S4+ZdGs0KSRcAxwJur6rtVdV9V3VtVX6iqj44zfsck30iyIcmmtrygb/1bk6xJcleS65P8Vas/P8kPktyR5NYkX3oSPT/q7Ga8Yyb5A+AzwH9ol9Bub2O3T3Jm6//GJB8YC4kkc5J8vPV3fZJjNzvO95OckOT/APcCz0tyZJJr2rHXJHlbX5+vbmeBf5fkliTrkxyW5JAk/55kY5L3P9E/B01PW00+RJoRDgDWVtUlA45/CvDPwJuAOcBy4B+Bw5I8HTgJeHlVXZtkF3pnQgAfBr4NvAbYBljcRfNbOmZVXZPk7cB/q6o/6tvkU8D2wPOAZ7ee1gOfA44GDgb2Bu4BvjzOId/SxlwLBHgh8J+ANcAfA99McmlV/aSNfw7wNGA+8FbgNOBCYF9gN2BlkrOq6von/6eh6cAzF80Wz6b3j+tAquq2qjq3nd3cBZwA/EnfkIeBvZJsW1Xrq2pVqz8A7A48t6p+U1WT3Qu5NcntYy/gLycYu6VjPkqbAHA48L6ququqbgA+Ti8woBeYn6yqtVW1CXjMmRtwelWtqqoHq+qBqjqvqn5ePT+gF1av6hv/AHBCVT0AnA3s3I5xV+vzauBlk/xZaAYxXDRb3AbsMujgJNsl+Wy7pHQn8ENghyRzquoe4C+AtwPrk5yX5EVt07+j9z/9S5KsSvJfJznUzlW1w9gL+OJ4gyY55mP2CWwN3NhXu5HeWQXAc4Gb+tb1L49bS3JwkovbJa7bgUPaccbc1jfx4Nft5819638NPGML/WoGMlw0W1wELEgy6GWq99C7FPSKqnoWvUtB0AsOquqCqnotvcD6Gb3LQFTVr6rq6Kp6LvA24OQkz+/iA2zpmMDmjza/lUfOoMbsBqxry+uBBX3rdh3vcGMLSZ4KnAv8PTCvheD5tD8LaTyGi2aFqroOOBk4q92A3ibJ05IcnuS4cTZ5Jr3/bd+eZCfg+LEVSeYlObTdB7kPuJveJSuSvLHvxv8mev9IP/xk+5/omPTOEBYk2aZ91oeAc4ATkjwzye7A3wL/0safA7wryfwkOwDvneTw2wBPBTYADyY5GDjwyX4mzWyGi2aTd9K7Kf9p4Hbg58DrgX8bZ+w/ANvSOwu4GPhW37qn0PvH+pfARnr3Yv57W/dy4MdJ7gZWAO+qqjUd9D7RMb8LrAJ+leTWVvsf9G7WrwF+RO9y2/K27jR690yuAH5K7yzkQWDc36dp95zeSS+UNtG7L7Sig8+kGSx+WZg0u7Uzkc9U1e6TDpYG5JmLNMsk2bb9DspWSebTu+T31anuSzOLZy7SLJNkO+AHwIvo3Vc6j97luzuntDHNKIaLJKlzXhaTJHXOcJEkdc5nizU777xzLVy4cKrbkKRp5bLLLru1quZuXjdcmoULF7Jy5cqpbkOSppUkN45X97KYJKlzhoskqXOGiySpc4aLJKlzhoskqXOGiySpc4aLJKlzhoskqXP+EqWkTiw87rypbmFGueGjr5vqFp4Uz1wkSZ0zXCRJnTNcJEmdM1wkSZ0zXCRJnTNcJEmdM1wkSZ0zXCRJnTNcJEmdM1wkSZ0zXCRJnTNcJEmdM1wkSZ0zXCRJnTNcJEmdM1wkSZ0zXCRJnTNcJEmdM1wkSZ0zXCRJnTNcJEmdG1q4JNk1yfeSXJ1kVZJ3tfoHk6xLcnl7HdK3zfuSrE5ybZKD+upLWm11kuP66nsk+XGrfynJNq3+1PZ+dVu/cFifU5L0WMM8c3kQeE9V7QnsDxyTZM+27sSq2ru9zgdo6w4HXgwsAU5OMifJHODTwMHAnsCb+/bzsbav5wObgKNa/ShgU6uf2MZJkkZkaOFSVeur6idt+S7gGmD+BJscCpxdVfdV1fXAamC/9lpdVWuq6n7gbODQJAH+FPhK2/4M4LC+fZ3Rlr8CHNDGS5JGYCT3XNplqT8EftxKxya5IsnyJDu22nzgpr7N1rbalurPBm6vqgc3qz9qX239HW385n0tS7IyycoNGzY8uQ8pSfqtoYdLkmcA5wLvrqo7gVOA3wf2BtYDHx92D1tSVadW1eKqWjx37typakOSZpyhhkuSrekFyxeq6l8Bqurmqnqoqh4GTqN32QtgHbBr3+YLWm1L9duAHZJstVn9Uftq67dv4yVJIzDM2WIBPgdcU1Wf6Kvv0jfs9cBVbXkFcHib6bUHsAi4BLgUWNRmhm1D76b/iqoq4HvAG9r2S4Gv9+1raVt+A/DdNl6SNAJbTT7kCXsl8BbgyiSXt9r76c322hso4AbgbQBVtSrJOcDV9GaaHVNVDwEkORa4AJgDLK+qVW1/7wXOTvIR4Kf0woz28/NJVgMb6QWSJGlEhhYuVfUjYLwZWudPsM0JwAnj1M8fb7uqWsMjl9X6678B3vh4+pUkdcff0Jckdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHVuaOGSZNck30tydZJVSd7V6jsluTDJde3njq2eJCclWZ3kiiT79O1raRt/XZKlffV9k1zZtjkpSSY6hiRpNIZ55vIg8J6q2hPYHzgmyZ7AccBFVbUIuKi9BzgYWNRey4BToBcUwPHAK4D9gOP7wuIU4Oi+7Za0+paOIUkagaGFS1Wtr6qftOW7gGuA+cChwBlt2BnAYW35UODM6rkY2CHJLsBBwIVVtbGqNgEXAkvaumdV1cVVVcCZm+1rvGNIkkZgJPdckiwE/hD4MTCvqta3Vb8C5rXl+cBNfZutbbWJ6mvHqTPBMSRJIzD0cEnyDOBc4N1VdWf/unbGUcM8/kTHSLIsycokKzds2DDMNiRpVhlquCTZml6wfKGq/rWVb26XtGg/b2n1dcCufZsvaLWJ6gvGqU90jEepqlOranFVLZ47d+4T+5CSpMcY5myxAJ8DrqmqT/StWgGMzfhaCny9r35EmzW2P3BHu7R1AXBgkh3bjfwDgQvaujuT7N+OdcRm+xrvGJKkEdhqiPt+JfAW4Mokl7fa+4GPAuckOQq4EXhTW3c+cAiwGrgXOBKgqjYm+TBwaRv3oara2JbfAZwObAt8s72Y4BiSpBEYWrhU1Y+AbGH1AeOML+CYLexrObB8nPpKYK9x6reNdwxJ0mj4G/qSpM4ZLpKkzhkukqTOGS6SpM4ZLpKkzhkukqTOGS6SpM4ZLpKkzg0ULkleMuxGJEkzx6BnLicnuSTJO5JsP9SOJEnT3kDhUlWvAv6K3tOJL0vyxSSvHWpnkqRpa+B7LlV1HfAB4L3AnwAnJflZkv88rOYkSdPToPdcXprkRHpfVfynwJ9V1R+05ROH2J8kaRoa9KnInwL+CXh/Vf16rFhVv0zygaF0JkmatgYNl9cBv66qhwCSPAV4WlXdW1WfH1p3kqRpadB7Lt+h94VcY7ZrNUmSHmPQcHlaVd099qYtbzecliRJ092g4XJPkn3G3iTZF/j1BOMlSbPYoPdc3g18Ockv6X118XOAvxhWU5Kk6W2gcKmqS5O8CHhhK11bVQ8Mry1J0nQ26JkLwMuBhW2bfZJQVWcOpStJ0rQ2ULgk+Tzw+8DlwEOtXIDhIkl6jEHPXBYDe1ZVDbMZSdLMMOhssavo3cSXJGlSg5657AxcneQS4L6xYlX9+VC6kiRNa4OGyweH2YQkaWYZdCryD5LsDiyqqu8k2Q6YM9zWJEnT1aCP3D8a+Arw2VaaD3xtkm2WJ7klyVV9tQ8mWZfk8vY6pG/d+5KsTnJtkoP66ktabXWS4/rqeyT5cat/Kck2rf7U9n51W79wkM8oSerOoDf0jwFeCdwJv/3isN+bZJvTgSXj1E+sqr3b63yAJHsChwMvbtucnGROkjnAp4GDgT2BN7exAB9r+3o+sAk4qtWPAja1+oltnCRphAYNl/uq6v6xN0m2ovd7LltUVT8ENg64/0OBs6vqvqq6HlgN7Ndeq6tqTTv+2cChSULvi8q+0rY/Azisb19ntOWvAAe08ZKkERk0XH6Q5P3AtkleC3wZ+LcneMxjk1zRLpvt2GrzgZv6xqxttS3Vnw3cXlUPblZ/1L7a+jvaeEnSiAwaLscBG4ArgbcB5wNP5BsoT6H3m/57A+uBjz+BfXQmybIkK5Os3LBhw1S2IkkzyqCzxR4GTmuvJ6yqbh5bTnIa8I32dh2wa9/QBa3GFuq3ATsk2aqdnfSPH9vX2nb5bvs2frx+TgVOBVi8eLFPH5Ckjgw6W+z6JGs2fz3egyXZpe/t6+n95j/ACuDwNtNrD2ARcAlwKbCozQzbht5N/xXtMTTfA97Qtl8KfL1vX0vb8huA7/rYGkkarcfzbLExTwPeCOw00QZJzgJeDeycZC1wPPDqJHvTmwxwA71LbFTVqiTnAFcDDwLHVNVDbT/HAhfQ+72a5VW1qh3ivcDZST4C/BT4XKt/Dvh8ktX0JhQcPuBnlCR1JE/0P/VJLquqfTvuZ8osXry4Vq5cOdVtSNPWwuPOm+oWZpQbPvq6qW5hIC0LFm9eH/SR+/v0vX0KvTOZx/NdMJKkWWTQgOif1fUgvUtab+q8G0nSjDDobLHXDLsRSdLMMehlsb+daH1VfaKbdiRJM8HjmS32cnrTfAH+jN5U4euG0ZQkaXobNFwWAPtU1V3Qe7oxcF5V/fWwGpMkTV+DPv5lHnB/3/v7W02SpMcY9MzlTOCSJF9t7w/jkScPS5L0KIPOFjshyTeBV7XSkVX10+G1JUmazga9LAawHXBnVX2S3kMh9xhST5KkaW7QB1ceT+9ZXu9rpa2BfxlWU5Kk6W3QM5fXA38O3ANQVb8EnjmspiRJ09ug4XJ/e2x9ASR5+vBakiRNd4OGyzlJPkvvC7qOBr7Dk/ziMEnSzDXpbLEkAb4EvAi4E3gh8L+q6sIh9yZJmqYmDZeqqiTnV9VLAANFkjSpQS+L/STJy4faiSRpxhj0N/RfAfx1khvozRgLvZOalw6rMUnS9DVhuCTZrap+ARw0on4kSTPAZGcuX6P3NOQbk5xbVf9lBD1Jkqa5ye65pG/5ecNsRJI0c0wWLrWFZUmStmiyy2IvS3InvTOYbdsyPHJD/1lD7U6SNC1NGC5VNWdUjUiSZo7H88h9SZIGYrhIkjpnuEiSOme4SJI6N7RwSbI8yS1Jruqr7ZTkwiTXtZ87tnqSnJRkdZIrkuzTt83SNv66JEv76vsmubJtc1J7evMWjyFJGp1hnrmcDizZrHYccFFVLQIuau8BDgYWtdcy4BToBQVwPL1nm+0HHN8XFqcAR/dtt2SSY0iSRmRo4VJVPwQ2blY+FDijLZ8BHNZXP7N6Lqb3pWS70Hum2YVVtbGqNtF75P+Stu5ZVXVx+4bMMzfb13jHkCSNyKjvucyrqvVt+VfAvLY8H7ipb9zaVpuovnac+kTHkCSNyJTd0G9nHEN9pMxkx0iyLMnKJCs3bNgwzFYkaVYZdbjc3C5p0X7e0urrgF37xi1otYnqC8apT3SMx6iqU6tqcVUtnjt37hP+UJKkRxt1uKwAxmZ8LQW+3lc/os0a2x+4o13augA4MMmO7Ub+gcAFbd2dSfZvs8SO2Gxf4x1DkjQig34T5eOW5Czg1cDOSdbSm/X1UeCcJEcBNwJvasPPBw4BVgP3AkcCVNXGJB8GLm3jPlRVY5ME3kFvRtq2wDfbiwmOIUkakaGFS1W9eQurDhhnbAHHbGE/y4Hl49RXAnuNU79tvGNIkkbH39CXJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1bkrCJckNSa5McnmSla22U5ILk1zXfu7Y6klyUpLVSa5Isk/ffpa28dclWdpX37ftf3XbNqP/lJI0e03lmctrqmrvqlrc3h8HXFRVi4CL2nuAg4FF7bUMOAV6YQQcD7wC2A84fiyQ2pij+7ZbMvyPI0ka87t0WexQ4Iy2fAZwWF/9zOq5GNghyS7AQcCFVbWxqjYBFwJL2rpnVdXFVVXAmX37kiSNwFSFSwHfTnJZkmWtNq+q1rflXwHz2vJ84Ka+bde22kT1tePUJUkjstUUHfePqmpdkt8DLkzys/6VVVVJathNtGBbBrDbbrsN+3CSNGtMyZlLVa1rP28BvkrvnsnN7ZIW7ectbfg6YNe+zRe02kT1BePUx+vj1KpaXFWL586d+2Q/liSpGXm4JHl6kmeOLQMHAlcBK4CxGV9Lga+35RXAEW3W2P7AHe3y2QXAgUl2bDfyDwQuaOvuTLJ/myV2RN++JEkjMBWXxeYBX22zg7cCvlhV30pyKXBOkqOAG4E3tfHnA4cAq4F7gSMBqmpjkg8Dl7ZxH6qqjW35HcDpwLbAN9tLkjQiIw+XqloDvGyc+m3AAePUCzhmC/taDiwfp74S2OtJNytJekJ+l6YiS5JmCMNFktQ5w0WS1DnDRZLUOcNFktQ5w0WS1DnDRZLUOcNFktQ5w0WS1DnDRZLUOcNFktQ5w0WS1DnDRZLUOcNFktQ5w0WS1DnDRZLUOcNFktQ5w0WS1DnDRZLUOcNFktQ5w0WS1DnDRZLUOcNFktQ5w0WS1DnDRZLUOcNFktQ5w0WS1DnDRZLUuRkbLkmWJLk2yeokx011P5I0m8zIcEkyB/g0cDCwJ/DmJHtObVeSNHvMyHAB9gNWV9WaqrofOBs4dIp7kqRZY6upbmBI5gM39b1fC7xi80FJlgHL2tu7k1w7gt5mi52BW6e6CWkc0+LvZj421R0MbPfxijM1XAZSVacCp051HzNRkpVVtXiq+5A259/N0Zipl8XWAbv2vV/QapKkEZip4XIpsCjJHkm2AQ4HVkxxT5I0a8zIy2JV9WCSY4ELgDnA8qpaNcVtzTZebtTvKv9ujkCqaqp7kCTNMDP1spgkaQoZLpKkzhkukqTOzcgb+pI0JsmL6D2hY34rrQNWVNU1U9fVzOeZi4YqyZFT3YNmryTvpff4pwCXtFeAs3yg7XA5W0xDleQXVbXbVPeh2SnJvwMvrqoHNqtvA6yqqkVT09nM52UxPWlJrtjSKmDeKHuRNvMw8Fzgxs3qu7R1GhLDRV2YBxwEbNqsHuD/jr4d6bfeDVyU5DoeeZjtbsDzgWOnqqnZwHBRF74BPKOqLt98RZLvj7wbqamqbyV5Ab2v4ei/oX9pVT00dZ3NfN5zkSR1ztlikqTOGS6SpM4ZLtIUSPKcJGcn+XmSy5Kcn+QFSa6a6t6kLnhDXxqxJAG+CpxRVYe32stw2rZmEM9cpNF7DfBAVX1mrFBV/49HpsqSZGGS/53kJ+31H1t9lyQ/THJ5kquSvCrJnCSnt/dXJvmb0X8k6dE8c5FGby/gsknG3AK8tqp+k2QRcBawGPhL4IKqOiHJHGA7YG9gflXtBZBkh2E1Lg3KcJF+N20N/GOSvYGHgBe0+qXA8iRbA1+rqsuTrAGel+RTwHnAt6eiYamfl8Wk0VsF7DvJmL8BbgZeRu+MZRuAqvoh8Mf0fhHw9CRHVNWmNu77wNuBfxpO29LgDBdp9L4LPDXJsrFCkpcCu/aN2R5YX1UPA28B5rRxuwM3V9Vp9EJknyQ7A0+pqnOBDwD7jOZjSFvmZTFpxKqqkrwe+If2SPjfADfQew7WmJOBc5McAXwLuKfVXw38zyQPAHcDR9B7rMk/Jxn7z+L7hv0ZpMn4+BdJUue8LCZJ6pzhIknqnOEiSeqc4SJJ6pzhIknqnOEiSeqc4SJJ6pzhIknq3P8HLFg+nTQ6CTYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "count_class.plot(kind='bar')\n",
    "plt.title(' Class Histogram')\n",
    "plt.xlabel('Class')\n",
    "plt.ylabel('Frequency')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Class = 1  只有492，\n",
    "Class = 0   有 284315， 数量相差大，样本不均衡，样本不规律\n",
    "\n",
    "解决方案：\n",
    "1. 过采样 （对小的样本，生成数量，和多的一样的多）\n",
    "2. 下采样 (取的时候，让多的和少的一样多)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>V1</th>\n",
       "      <th>V2</th>\n",
       "      <th>V3</th>\n",
       "      <th>V4</th>\n",
       "      <th>V5</th>\n",
       "      <th>V6</th>\n",
       "      <th>V7</th>\n",
       "      <th>V8</th>\n",
       "      <th>V9</th>\n",
       "      <th>V10</th>\n",
       "      <th>...</th>\n",
       "      <th>V21</th>\n",
       "      <th>V22</th>\n",
       "      <th>V23</th>\n",
       "      <th>V24</th>\n",
       "      <th>V25</th>\n",
       "      <th>V26</th>\n",
       "      <th>V27</th>\n",
       "      <th>V28</th>\n",
       "      <th>Class</th>\n",
       "      <th>normAmount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1.359807</td>\n",
       "      <td>-0.072781</td>\n",
       "      <td>2.536347</td>\n",
       "      <td>1.378155</td>\n",
       "      <td>-0.338321</td>\n",
       "      <td>0.462388</td>\n",
       "      <td>0.239599</td>\n",
       "      <td>0.098698</td>\n",
       "      <td>0.363787</td>\n",
       "      <td>0.090794</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.018307</td>\n",
       "      <td>0.277838</td>\n",
       "      <td>-0.110474</td>\n",
       "      <td>0.066928</td>\n",
       "      <td>0.128539</td>\n",
       "      <td>-0.189115</td>\n",
       "      <td>0.133558</td>\n",
       "      <td>-0.021053</td>\n",
       "      <td>0</td>\n",
       "      <td>0.244964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.191857</td>\n",
       "      <td>0.266151</td>\n",
       "      <td>0.166480</td>\n",
       "      <td>0.448154</td>\n",
       "      <td>0.060018</td>\n",
       "      <td>-0.082361</td>\n",
       "      <td>-0.078803</td>\n",
       "      <td>0.085102</td>\n",
       "      <td>-0.255425</td>\n",
       "      <td>-0.166974</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.225775</td>\n",
       "      <td>-0.638672</td>\n",
       "      <td>0.101288</td>\n",
       "      <td>-0.339846</td>\n",
       "      <td>0.167170</td>\n",
       "      <td>0.125895</td>\n",
       "      <td>-0.008983</td>\n",
       "      <td>0.014724</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.342475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.358354</td>\n",
       "      <td>-1.340163</td>\n",
       "      <td>1.773209</td>\n",
       "      <td>0.379780</td>\n",
       "      <td>-0.503198</td>\n",
       "      <td>1.800499</td>\n",
       "      <td>0.791461</td>\n",
       "      <td>0.247676</td>\n",
       "      <td>-1.514654</td>\n",
       "      <td>0.207643</td>\n",
       "      <td>...</td>\n",
       "      <td>0.247998</td>\n",
       "      <td>0.771679</td>\n",
       "      <td>0.909412</td>\n",
       "      <td>-0.689281</td>\n",
       "      <td>-0.327642</td>\n",
       "      <td>-0.139097</td>\n",
       "      <td>-0.055353</td>\n",
       "      <td>-0.059752</td>\n",
       "      <td>0</td>\n",
       "      <td>1.160686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.966272</td>\n",
       "      <td>-0.185226</td>\n",
       "      <td>1.792993</td>\n",
       "      <td>-0.863291</td>\n",
       "      <td>-0.010309</td>\n",
       "      <td>1.247203</td>\n",
       "      <td>0.237609</td>\n",
       "      <td>0.377436</td>\n",
       "      <td>-1.387024</td>\n",
       "      <td>-0.054952</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.108300</td>\n",
       "      <td>0.005274</td>\n",
       "      <td>-0.190321</td>\n",
       "      <td>-1.175575</td>\n",
       "      <td>0.647376</td>\n",
       "      <td>-0.221929</td>\n",
       "      <td>0.062723</td>\n",
       "      <td>0.061458</td>\n",
       "      <td>0</td>\n",
       "      <td>0.140534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.158233</td>\n",
       "      <td>0.877737</td>\n",
       "      <td>1.548718</td>\n",
       "      <td>0.403034</td>\n",
       "      <td>-0.407193</td>\n",
       "      <td>0.095921</td>\n",
       "      <td>0.592941</td>\n",
       "      <td>-0.270533</td>\n",
       "      <td>0.817739</td>\n",
       "      <td>0.753074</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.009431</td>\n",
       "      <td>0.798278</td>\n",
       "      <td>-0.137458</td>\n",
       "      <td>0.141267</td>\n",
       "      <td>-0.206010</td>\n",
       "      <td>0.502292</td>\n",
       "      <td>0.219422</td>\n",
       "      <td>0.215153</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.073403</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         V1        V2        V3        V4        V5        V6        V7  \\\n",
       "0 -1.359807 -0.072781  2.536347  1.378155 -0.338321  0.462388  0.239599   \n",
       "1  1.191857  0.266151  0.166480  0.448154  0.060018 -0.082361 -0.078803   \n",
       "2 -1.358354 -1.340163  1.773209  0.379780 -0.503198  1.800499  0.791461   \n",
       "3 -0.966272 -0.185226  1.792993 -0.863291 -0.010309  1.247203  0.237609   \n",
       "4 -1.158233  0.877737  1.548718  0.403034 -0.407193  0.095921  0.592941   \n",
       "\n",
       "         V8        V9       V10  ...       V21       V22       V23       V24  \\\n",
       "0  0.098698  0.363787  0.090794  ... -0.018307  0.277838 -0.110474  0.066928   \n",
       "1  0.085102 -0.255425 -0.166974  ... -0.225775 -0.638672  0.101288 -0.339846   \n",
       "2  0.247676 -1.514654  0.207643  ...  0.247998  0.771679  0.909412 -0.689281   \n",
       "3  0.377436 -1.387024 -0.054952  ... -0.108300  0.005274 -0.190321 -1.175575   \n",
       "4 -0.270533  0.817739  0.753074  ... -0.009431  0.798278 -0.137458  0.141267   \n",
       "\n",
       "        V25       V26       V27       V28  Class  normAmount  \n",
       "0  0.128539 -0.189115  0.133558 -0.021053      0    0.244964  \n",
       "1  0.167170  0.125895 -0.008983  0.014724      0   -0.342475  \n",
       "2 -0.327642 -0.139097 -0.055353 -0.059752      0    1.160686  \n",
       "3  0.647376 -0.221929  0.062723  0.061458      0    0.140534  \n",
       "4 -0.206010  0.502292  0.219422  0.215153      0   -0.073403  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import  StandardScaler\n",
    "# ‘Amount’列的值明显比其他V1到V28要大得多多，这个时候为了避免出现因为特征值的差距过大导致模型的判断失误，我们需要对数据进行预处理，在大多数情况下，我们进行数据分析前，都需要先进行数据的预处理。\n",
    "# Data['Amount'] = (Data['Amount'] -Data['Amount'].min())/(Data['Amount'].max()-Data['Amount'].min())\n",
    "\n",
    "# 由于Amount数值比其他属性明显过大， 采取归一化\n",
    "data['normAmount'] = StandardScaler().fit_transform(data['Amount'].values.reshape(-1, 1))\n",
    "data = data.drop(['Time','Amount'], axis=1)\n",
    "data.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 下采样策略"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "X = data.loc[:, data.columns != 'Class']\n",
    "Y = data.loc[:, data.columns == 'Class']\n",
    "number_record_fraud = len(data[data.Class == 1])\n",
    "\n",
    "fraud_indices = np.array(data[data.Class == 1].index)\n",
    "normal_indices = np.array(data[data.Class == 0].index)\n",
    "\n",
    "random_normal_indices = np.random.choice(normal_indices, number_record_fraud, replace=False)\n",
    "random_normal_indices = np.array(random_normal_indices)\n",
    "\n",
    "# append 2 indices\n",
    "under_sample_indices = np.concatenate([fraud_indices, random_normal_indices])\n",
    "\n",
    "under_sample_data = data.iloc[under_sample_indices,:]\n",
    "X_under_sample = under_sample_data.loc[:, under_sample_data.columns != 'Class']\n",
    "Y_under_sample = under_sample_data.loc[:, under_sample_data.columns == 'Class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    492\n",
       "0    492\n",
       "Name: Class, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "under_sample_data.Class.value_counts() # 查看数据"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 交叉验证\n",
    "\n",
    "1. 数据切分 80%去训练 20%去测试\n",
    "2. 训练集的切换，等分3份； （1）1 ，2 去建立训练，3 去验证模型 （2，3）轮流更换验证集\n",
    "3. 可以使用3次参数的平均 来平衡模型\n",
    "4.\n",
    "\n",
    "[sklearn之train_test_split()函数各参数含义（非常全）](https://www.cnblogs.com/Yanjy-OnlyOne/p/11288098.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number train: 199364\n",
      "number test: 85443\n",
      "number train: 688\n",
      "number test: 296\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# whole dataset 总数据\n",
    "X_train, X_test, Y_train,Y_test=train_test_split(X, Y, test_size=0.3, random_state=0)\n",
    "print('number train:', len(X_train))\n",
    "print('number test:', len(X_test))\n",
    "\n",
    "# 下采样 dataset 总数据\n",
    "X_train_under_sample, X_test_under_sample, Y_train_under_sample,Y_test_under_sample=train_test_split(X_under_sample, Y_under_sample, test_size=0.3, random_state=0)\n",
    "\n",
    "print('number train:', len(X_train_under_sample))\n",
    "print('number test:', len(X_test_under_sample))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 模型评估\n",
    "\n",
    "精度\n",
    "\n",
    "Recall 召回率/查全率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Recall = TP / (TP+FN)\n",
    "from sklearn.linear_model import  LogisticRegression\n",
    "from sklearn.model_selection import KFold, cross_val_score\n",
    "from sklearn.metrics import confusion_matrix,recall_score,classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def printing_kfold_score(x_train_data, y_train_data):\n",
    "  fold = KFold(n_splits=5, shuffle=False) # 做几次交叉验证。把训练集切成多分不仅是3， 此处切分成5份\n",
    "\n",
    "  # different c parameter\n",
    "  c_param_range = [0.01,0.1,1,100,100] # 正则化惩罚项\n",
    "  # 希望模型更泛化，参数更稳定。\n",
    "  # 过拟合：训练集合表达好，在测试机表现不好\n",
    "  # L1 L2 正则方法\n",
    "  results_table = pd.DataFrame(index=range(len(c_param_range),2), columns=['C_Parameter','Mean recall score'])\n",
    "  results_table['C_Parameter'] = c_param_range\n",
    "\n",
    "  j = 0\n",
    "  for c_param in c_param_range:\n",
    "    print('-'*30)\n",
    "    print('c_param:',c_param)\n",
    "    print('-'*30)\n",
    "    print('')\n",
    "    recall_accs = []\n",
    "    i = 1\n",
    "    for train_index , test_index in fold.split(x_train_data):\n",
    "#     for iteration,indices in enumerate(fold, start=1):\n",
    "      # call logisitic regresstion mode\n",
    "      lr = LogisticRegression(C = c_param, penalty='l1',solver='liblinear') # 使用L1\n",
    "\n",
    "      lr.fit(x_train_data.iloc[train_index,:], y_train_data.iloc[train_index,:].values.ravel())\n",
    "\n",
    "      y_pred_under_sample = lr.predict(x_train_data.iloc[test_index,:].values) # 去验证\n",
    "\n",
    "      recall_acc = recall_score(y_train_data.iloc[test_index,:].values, y_pred_under_sample)\n",
    "      recall_accs.append(recall_acc)\n",
    "      print('iteration: ',i, ',recall_score:', recall_acc)\n",
    "      i += 1\n",
    "\n",
    "    results_table.loc[j, 'Mean recall score'] = np.mean(recall_accs)\n",
    "    j += 1\n",
    "    print()\n",
    "    print('Mean recall score', np.mean(recall_accs))\n",
    "    print()\n",
    "  idx = results_table['Mean recall score'].astype(float).idxmax()\n",
    "  best_c = results_table.loc[idx]['C_Parameter']\n",
    "\n",
    "  print('*'*30)\n",
    "  print('best_c:',best_c)\n",
    "  print('*'*30)\n",
    "  return best_c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------\n",
      "c_param: 0.01\n",
      "------------------------------\n",
      "\n",
      "iteration:  1 ,recall_score: 0.9315068493150684\n",
      "iteration:  2 ,recall_score: 0.9178082191780822\n",
      "iteration:  3 ,recall_score: 1.0\n",
      "iteration:  4 ,recall_score: 0.9594594594594594\n",
      "iteration:  5 ,recall_score: 0.9696969696969697\n",
      "\n",
      "Mean recall score 0.955694299529916\n",
      "\n",
      "------------------------------\n",
      "c_param: 0.1\n",
      "------------------------------\n",
      "\n",
      "iteration:  1 ,recall_score: 0.8493150684931506\n",
      "iteration:  2 ,recall_score: 0.863013698630137\n",
      "iteration:  3 ,recall_score: 0.9491525423728814\n",
      "iteration:  4 ,recall_score: 0.9459459459459459\n",
      "iteration:  5 ,recall_score: 0.9090909090909091\n",
      "\n",
      "Mean recall score 0.9033036329066049\n",
      "\n",
      "------------------------------\n",
      "c_param: 1\n",
      "------------------------------\n",
      "\n",
      "iteration:  1 ,recall_score: 0.863013698630137\n",
      "iteration:  2 ,recall_score: 0.8904109589041096\n",
      "iteration:  3 ,recall_score: 0.9661016949152542\n",
      "iteration:  4 ,recall_score: 0.9459459459459459\n",
      "iteration:  5 ,recall_score: 0.8939393939393939\n",
      "\n",
      "Mean recall score 0.9118823384669682\n",
      "\n",
      "------------------------------\n",
      "c_param: 100\n",
      "------------------------------\n",
      "\n",
      "iteration:  1 ,recall_score: 0.863013698630137\n",
      "iteration:  2 ,recall_score: 0.9041095890410958\n",
      "iteration:  3 ,recall_score: 0.9830508474576272\n",
      "iteration:  4 ,recall_score: 0.9459459459459459\n",
      "iteration:  5 ,recall_score: 0.9090909090909091\n",
      "\n",
      "Mean recall score 0.921042198033143\n",
      "\n",
      "------------------------------\n",
      "c_param: 100\n",
      "------------------------------\n",
      "\n",
      "iteration:  1 ,recall_score: 0.863013698630137\n",
      "iteration:  2 ,recall_score: 0.9041095890410958\n",
      "iteration:  3 ,recall_score: 0.9830508474576272\n",
      "iteration:  4 ,recall_score: 0.9459459459459459\n",
      "iteration:  5 ,recall_score: 0.9090909090909091\n",
      "\n",
      "Mean recall score 0.921042198033143\n",
      "\n",
      "******************************\n",
      "best_c: 0.01\n",
      "******************************\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.01"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_c = printing_kfold_score(X_train_under_sample, Y_train_under_sample)\n",
    "best_c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "混淆矩阵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import itertools\n",
    "def plot_confusion_matrix(cm, labels, title='Confusion Matrix', cmap=plt.cm.Blues):\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "\n",
    "    tick_marks = np.arange(len(labels))\n",
    "    plt.xticks(tick_marks, labels, rotation=0)\n",
    "    plt.yticks(tick_marks, labels)\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "\n",
    "    thresh = cm.max() /2\n",
    "    for i,j in itertools.product(range(cm.shape[0]),range(cm.shape[1])):\n",
    "      plt.text(j,i,cm[i,j], horizontalalignment='center', color= 'white' if cm[i,j] > thresh else 'black')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall metric in test:  0.9319727891156463\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEWCAYAAADy2YssAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAc20lEQVR4nO3de7xVZb3v8c8XULyiXBRJQd1KGqGYkuKNSPcuSAo7L/OGiqaHLM1O6XFrFy+ZOyuzrKN5SE0EE9EyQRH1sGUr7jTB0FBTCUVR5Gqkghfid/4YY+FkuS5zTOZcc47B9+1rvtaczxjrGb+1gK/PM66KCMzMiqhTvQswM6sVB5yZFZYDzswKywFnZoXlgDOzwnLAmVlhOeAKRtKWkqZKWiXp9o3oZ7Sk+6tZWz1IulfSmHrXYfXhgKsTSSdKmi3pLUmL03+Ih1Wh62OA3kDPiPhSpZ1ExC0R8Zkq1LMBScMkhaQ7m7UPSttnltnPJZImtrdeRIyIiPEVlms554CrA0nfAn4O/AdJGPUDrgVGVaH7XYHnI2JtFfqqlWXAwZJ6lrSNAZ6v1gaU8N/vTV1E+NWBL2A74C3gS22s05UkAF9LXz8HuqbLhgGLgHOBpcBi4LR02aXAe8D76TZOBy4BJpb0vRsQQJf086nAAuBN4EVgdEn7rJLvOwR4HFiVfj2kZNlM4DLgkbSf+4FerfxsTfVfB5yVtnUGXgUuAmaWrHs18ArwD2AOcHjaPrzZz/lkSR2Xp3WsAfZM285Il/8K+F1J/z8CZgCq998Lv2rz8v/hOt7BwBbAnW2s8x1gCLAfMAg4EPhuyfKdSIJyZ5IQu0ZS94i4mGRUeFtEbBMRN7RViKStgV8AIyJiW5IQm9vCej2Ae9J1ewJXAfc0G4GdCJwG7AhsDpzX1raBm4FT0vefBeaRhHmpx0l+Bz2A3wK3S9oiIqY3+zkHlXzPycBYYFtgYbP+zgX2kXSqpMNJfndjIk07Kx4HXMfrCSyPtqeQo4HvR8TSiFhGMjI7uWT5++ny9yNiGskoZq8K61kHDJS0ZUQsjoinW1jnKOCFiJgQEWsj4lbgr8DnS9b5TUQ8HxFrgMkkwdSqiPhvoIekvUiC7uYW1pkYESvSbf6UZGTb3s95U0Q8nX7P+836W03ye7wKmAh8PSIWtdOf5ZgDruOtAHpJ6tLGOh9hw9HHwrRtfR/NAnI1sE3WQiLibeA44ExgsaR7JO1dRj1NNe1c8vn1CuqZAJwNfJoWRrSSzpP0bHpE+O8ko9Ze7fT5SlsLI+Ixkim5SILYCswB1/H+CLwLHN3GOq+RHCxo0o8PT9/K9TawVcnnnUoXRsR9EfFvQB+SUdmvy6inqaZXK6ypyQTga8C0dHS1XjqFPB84FugeEduT7P9TU+mt9NnmdFPSWSQjwdfS/q3AHHAdLCJWkexMv0bS0ZK2krSZpBGSfpyudivwXUk7SOqVrt/uKRGtmAsMldRP0nbAhU0LJPWWNCrdF/cuyVR3XQt9TAM+mp7a0kXSccAA4O4KawIgIl4EPkWyz7G5bYG1JEdcu0i6COhWsnwJsFuWI6WSPgr8ADiJZKp6vqT9Kqve8sABVwfp/qRvkRw4WEYyrTob+EO6yg+A2cBTwF+AJ9K2Srb1AHBb2tccNgylTmkdrwErScLmqy30sQIYSbKTfgXJyGdkRCyvpKZmfc+KiJZGp/cB00lOHVkIvMOG08+mk5hXSHqive2kuwQmAj+KiCcj4gXg28AESV035mewxiUfQDKzovIIzswKywFnZoXlgDOzwnLAmVlhtXWyaYdTly1DXbu1v6I1jEF79a13CZbByy+/xIrly9X+mq3r3G3XiLVrylo31iy7LyKGb8z2NkZjBVzXbnQdMLreZVgGMx/6Sb1LsAyGHXrQRvcRa9fQda9jy1r3nbnXtHflSU01VMCZWR4IcnInKgecmWUjoFPneldRFgecmWWnjdqN12EccGaWkaeoZlZkHsGZWSEJj+DMrKjkEZyZFZiPoppZMfkgg5kVlfAU1cwKzCM4MysmT1HNrKgEdPZBBjMrKu+DM7Ni8hTVzIrMIzgzKyyP4MyskORLtcysyHyplpkVkw8ymFmReYpqZoXk+8GZWXF5impmRZaTgwz5iGEzayxNp4q092q3G90oaamkeSVtP5H0V0lPSbpT0vYlyy6UNF/Sc5I+217/Djgzy0bpFLWcV/tuAoY3a3sAGBgR+wLPAxcmm9UA4Hjg4+n3XCupzaGkA87MsqvSCC4iHgJWNmu7PyLWph8fBXZJ348CJkXEuxHxIjAfOLCt/h1wZpaZpLJeQC9Js0teYzNu6svAven7nYFXSpYtStta5YMMZpZJcsfyss+DWx4RgyvajvQdYC1wSyXfDw44M8tKQp1qe6KvpFOBkcCRERFp86tA35LVdknbWuUpqplllmGKWknfw4HzgS9ExOqSRVOA4yV1lbQ70B/4U1t9eQRnZplVGl4t9HMrMIxkX90i4GKSo6ZdgQfS7TwaEWdGxNOSJgPPkExdz4qIf7bVvwPOzDKrVsBFxAktNN/QxvqXA5eX278DzsyyUfrKAQecmWUiKt+/1tEccGaWWadO+Tg+6YAzs8w8gjOzYvI+ODMrMo/gzKyQfJDBzAqt1pdqVYsDzsyykaeoZlZgDjgzKywHnJkVkg8ymFmx5SPffD+4arjue8ex8L5LmT3pf69vu+jM4fzpt+fx6C3nMvWXX6FPr24AHL7/Hrz+4OU8esu5PHrLuVx4xmfqVbYBZ33lDPbctQ8HDx70oWW/vPoqtt+qCyuWL69DZQ1MyaVa5bzqrf4VFMCEux9n1DnjNmj72YQHOfDEKxky+qfcO+uZDYLskT8vYMjonzJk9E/54fX3d3S5VuLEk0/hjj/c86H2RYte4cEZD7BL3351qKrx1fKGl9XkgKuCR/68gJX/WL1B25tvv7v+/VZbbs4Hd122RnLoYUPp3qPHh9q/ff65XPqDKxriH2lDUpmvOvM+uBq65KsjGH3UYFa99Q7Dz7x2fftB++zGY7ecx+Llq7jw6ik8u2BJHau05u6ZOoU+H9mZffb98LTVEnkJ/pqO4CQNT59APV/SBbXcViO65Ff30n/kZUya/gRnHnsYAHOfW8ReX7iMg0Zfya9um8Xkn3y5zlVaqdWrV3PVT37It793Sb1LaVjlTk8bIQRrFnDpE6evAUYAA4AT0idTb3Juu3cORx+xL5BMXd9e8x4A9/33s2zWpTM9t9u6nuVZiRcX/I2FC1/isIP2Z5+99+C1VxfxqUM+yZLXX693aQ0lLwFXyynqgcD8iFgAIGkSyZOpn6nhNhvGHn178bdXkqNvIz81kOdfWgpA757bsmTFmwAMHtCPTp3EilVv161O29DHB+7D/IWL13/eZ+89mDnrMXr26lXHqhqPr0Vt+SnUBzVfKX3SdfK06823rWE5tTP+Bydx+AF70mv7rZl/90VcNu4+hh/6MfrvugPr1gUvv/4G5/zwDgC+eMQg/ucxh7B27Treefd9TvnOhDpXv2k7fcxoZj30X6xYsZwBe+7KBd+9mFNO9W6D9jTC6KwcdT/IEBHjgHEAnbbunctDjWO+O/FDbeOnPNbiutfdPovrbp9V65KsTDeMb/uh6X/56986qJIc8cX2QAVPoTazxicgJ/lW06OojwP9Je0uaXPgeJInU5tZruXnKGrNRnARsVbS2cB9QGfgxoh4ulbbM7OO08kHGSAipgHTarkNM+tg8hTVzApKJCO4cl7t9iXdKGmppHklbT0kPSDphfRr97Rdkn6RXjjwlKT92+vfAWdmmUnlvcpwEzC8WdsFwIyI6A/MSD9DctFA//Q1FvhVe5074Mwss2odZIiIh4CVzZpHAePT9+OBo0vab47Eo8D2kvq01X/dz4Mzs5zJtg+ul6TZJZ/Hpee+tqV3RDRdTvI60Dt939LFAzsDi2mFA87MMhHKcjPL5RExuNJtRURIqvgCAE9RzSyzKu6Da8mSpqln+nVp2p754gEHnJllVuMTfacAY9L3Y4C7StpPSY+mDgFWlUxlW+QpqpllU8Xz4CTdCgwj2Ve3CLgYuAKYLOl0YCFwbLr6NOBzwHxgNXBae/074Mwsk+Ra1OokXESc0MqiI1tYN4CzsvTvgDOzzPJyJYMDzswy87WoZlZMvh+cmRVVnu4H54Azs4wa415v5XDAmVlmOck3B5yZZSQfZDCzgqrmeXC15oAzs8wccGZWWDnJNwecmWXnEZyZFVOOHjrjgDOzTJIbXuYj4RxwZpZZp5wM4RxwZpZZTvLNAWdm2cgX25tZkeVkF1zrASfpl0CrT7OJiHNqUpGZNbwiHGSY3cYyM9tEieRIah60GnARMb70s6StImJ17Usys0aXkwFc+48NlHSwpGeAv6afB0m6tuaVmVljKvORgY1wIKKc56L+HPgssAIgIp4EhtawJjNrcDV+8HPVlHUUNSJeaZbG/6xNOWbW6ESxTvR9RdIhQEjaDPgG8GxtyzKzRpaXo6jlTFHPJHnY6s7Aa8B+ZHz4qpkVR7nT00YY5LU7gouI5cDoDqjFzHKiWlNUSd8EziA55/YvwGlAH2AS0BOYA5wcEe9VVGcZBfyLpKmSlklaKukuSf9SycbMrBhU5qvNPqSdgXOAwRExEOgMHA/8CPhZROwJvAGcXmmd5UxRfwtMJknVjwC3A7dWukEzy78qnibSBdhSUhdgK2AxcARwR7p8PHB0pXWWE3BbRcSEiFibviYCW1S6QTPLt+QoankvoJek2SWvsU39RMSrwJXAyyTBtopkSvr3iFibrraIZP9/Rdq6FrVH+vZeSReQzIkDOA6YVukGzSznlOmGl8sjYnDL3ag7MArYHfg7yexweDVKbNLWQYY5JIHW9JN8pWRZABdWsxAzy48qXaXwr8CLEbEs7fP3wKHA9pK6pKO4XYBXK91AW9ei7l5pp2ZWXE1T1Cp4GRgiaStgDXAkyU0+HgSOIZk1jgHuqnQDZV3JIGkgMICSfW8RcXOlGzWzfKvGCC4iHpN0B/AEsBb4MzAOuAeYJOkHadsNlW6j3YCTdDEwjCTgpgEjgFmAA85sE1Wtc3gj4mLg4mbNC4ADq9F/OUdRjyEZOr4eEacBg4DtqrFxM8sfCTp3UlmveitniromItZJWiupG7AU6FvjusysgTXCrZDKUU7AzZa0PfBrkiOrbwF/rGVRZtbYcpJvZV2L+rX07XWSpgPdIuKp2pZlZo1KKP+3S5K0f1vLIuKJ2pRkZg2tQe4UUo62RnA/bWNZkFwvVlWf2Lsvj/zxqmp3azXU/ZNn17sEy+Dd516uSj+53wcXEZ/uyELMLB8EdM57wJmZtaYBzgApiwPOzDJzwJlZISW3I89HwpVzR19JOknSRennfpKqchmFmeVThvvB1bfOMta5FjgYOCH9/CZwTc0qMrOGV5iHzgAHRcT+kv4MEBFvSNq8xnWZWYMS0KUR0qsM5QTc+5I6k5z7hqQdgHU1rcrMGlpO8q2sgPsFcCewo6TLSe4u8t2aVmVmDUsqwKVaTSLiFklzSG6ZJODoiPCT7c02YTnJt7JueNkPWA1MLW2LiOpc82FmudMIR0jLUc4U9R4+ePjMFiRPwHkO+HgN6zKzBiVoiJtZlqOcKeo+pZ/Tu4x8rZXVzazoGuQct3JkvpIhIp6QdFAtijGzfFDVnspQW+Xsg/tWycdOwP7AazWryMwaWhUfG1hz5Yzgti15v5Zkn9zvalOOmeVBIQIuPcF324g4r4PqMbMcyMvF9m3dsrxLRKyVdGhHFmRmjS15bGC9qyhPWyO4P5Hsb5sraQpwO/B208KI+H2NazOzBlWYKxlIzn1bQfIMhqbz4QJwwJltgqp5kCF9JOn1wECSXPkyyXm2twG7AS8Bx0bEG5X039ZAc8f0COo84C/p16fTr/Mq2ZiZFUMVb5d0NTA9IvYGBgHPAhcAMyKiPzAj/VyRtkZwnYFtoMUTXqLSDZpZ3olOVTgPTtJ2wFDgVICIeA94T9IoYFi62nhgJvDvlWyjrYBbHBHfr6RTMysukeli+16SZpd8HhcR49L3uwPLgN9IGgTMAb4B9I6Ixek6rwO9K621rYDLx15EM+tYgi7l74RbHhGDW1nWheRA5tcj4jFJV9NsOhoRIaniGWNb++COrLRTMyuuphFcFfbBLQIWRcRj6ec7SAJviaQ+AOnXpZXW2mrARcTKSjs1s2LrlN70sr1XWyLideAVSXulTUcCzwBTgDFp2xjgrkrr9GMDzSyzKp4G93XglvQ5LwuA00gGXpMlnQ4sBI6ttHMHnJllIsp7HF85ImIu0NI+uqrsInPAmVk2KtaVDGZm6yVXMjjgzKyg8hFvDjgzq0BOBnAOODPLSvm/H5yZWUuqeRS11hxwZpaZDzKYWTGpALcsNzNriaeoZlZoHsGZWWHlI94ccGaWkYDOHsGZWVHlJN8ccGaWlVBOJqkOODPLzCM4Myuk5DSRfCScA87Msin/mad154Azs8x8qdYm6itnfJl7p93NDjvuyJy58wBYuXIlJ594HAsXvsSuu+7GxFsn07179zpXuum67uLRjBg6kGUr32Twl/4DgIu+dhQjP7Uv6yJYtvJNxl48kcXLVvHNU47kuM99EoAunTux9+470feIC3jjH6vr+SPUVXLDy3pXUZ68XHGRGyePOZW77p6+QduVP76CYUccybxnX2DYEUdy5Y+vqFN1BjBh6qOMOuuaDdp+Nn4GBx73Q4YcfwX3PjyPC8eOSNpvnsGQ469gyPFXcNEvp/DwnBc26XBrojL/qzcHXJUddvhQevTosUHb3VPv4qSTk6egnXTyGKZO+UMdKrMmjzzxN1au2jCk3nz7nfXvt9qyKxEfftbwscMHM3n6nJrXlwdVei5qzXmK2gGWLllCnz59ANhpp51YumRJnSuyllxy1ucZPfJAVr21huFjf7HBsi232Ix/O+RjfPOKyXWqrrE0wuisHDUbwUm6UdJSSfNqtY08kvJzN9RNzSXXTKX/iO8x6d7ZnHnc0A2WHTV0H/44d4Gnp3ywD66cV73Vcop6EzC8hv3nxo69e7N48WIAFi9ezA477ljniqwtt017nKOP3G+Dti999gBu9/Q0UeZT7RvhSGvNAi4iHgJW1qr/PDlq5BeYOGE8ABMnjGfk50fVuSJrbo9+O6x/P3LYvjz/0ge7EbptswWHHbAnU2c+VY/SGpLKfNVb3ffBSRoLjAXo269fnavZeKecdAIP/9dMli9fzh677cL3LrqU886/gJNOOJbxv7mBfv12ZeKt3o9TT+N/eCqHH9CfXttvw/zpl3HZddMYftjH6b/rjqxbF7y8eCXnXD5p/fpf+PQgZjz6V1a/814dq24c1X4uqqTOwGzg1YgYKWl3YBLQE5gDnBwRFf3y1dLRomqRtBtwd0QMLGf9Aw4YHI88Nrtm9Vj1df/k2fUuwTJ497nJrFu9dKPS6WP7fCJ+c+eDZa17cP/ucyJicFvrSPoWMBjolgbcZOD3ETFJ0nXAkxHxq0pq9WkiZpZdleaoknYBjgKuTz8LOAK4I11lPHB0pWXWfYpqZvmTYYraS1LptGxcRIwr+fxz4Hxg2/RzT+DvEbE2/bwI2LnSOmsWcJJuBYaR/ICLgIsj4oZabc/MOk6GOe7y1qaokkYCSyNijqRhVSmsmZoFXEScUKu+zazOqnOM4VDgC5I+B2wBdAOuBraX1CUdxe0CvFrpBrwPzswySXavbfy1qBFxYUTsEhG7AccD/xkRo4EHgWPS1cYAd1VaqwPOzLIp8zrUjTiT5N+Bb0maT7JPruJdWz7IYGaZVfsk3oiYCcxM3y8ADqxGvw44M8soP9dTO+DMLLOc5JsDzsyyaZTrTMvhgDOz7HKScA44M8ssLze8dMCZWWbeB2dmxdQgz1sohwPOzDLzFNXMCkl4BGdmBZaTfHPAmVkFcpJwDjgzy6wRnphVDgecmWWWj3hzwJlZJXKScA44M8uk6YaXeeCAM7NsfKKvmRVZTvLNAWdmWfmGl2ZWYDnJNwecmWXjG16aWbHlJOEccGaWmU8TMbPC8j44MysmQScHnJkVVz4SzgFnZpnk6YaXnepdgJnlj8p8tdmH1FfSg5KekfS0pG+k7T0kPSDphfRr90rrdMCZWWZSea92rAXOjYgBwBDgLEkDgAuAGRHRH5iRfq6IA87MMpNU1qstEbE4Ip5I378JPAvsDIwCxqerjQeOrrRO74Mzs8wy7ILrJWl2yedxETHuQ/1JuwGfAB4DekfE4nTR60DvSut0wJlZJmVOP5ssj4jBbfenbYDfAf8rIv5ROvKLiJAUldbqKaqZZaYy/2u3H2kzknC7JSJ+nzYvkdQnXd4HWFppnQ44M8uuCodRlQzVbgCejYirShZNAcak78cAd1VapqeoZpZZlU6DOxQ4GfiLpLlp27eBK4DJkk4HFgLHVroBB5yZZaSqPDYwImbRelYeudEbwAFnZhn5SgYzswbgEZyZZZaXEZwDzswy8w0vzayY/FxUMyuqPB1kcMCZWWaeoppZYXkEZ2aFlZN8c8CZWQVyknAOODPLRFCVS7U6giIqvtVS1UlaRnJxbdH0ApbXuwjLpKh/ZrtGxA4b04Gk6SS/n3Isj4jhG7O9jdFQAVdUkma3d9M/ayz+MysGX4tqZoXlgDOzwnLAdYwPPWTDGp7/zArA++DMrLA8gjOzwnLAmVlhOeBqSNJwSc9Jmi/pgnrXY+2TdKOkpZLm1bsW23gOuBqR1Bm4BhgBDABOkDSgvlVZGW4C6nZiqlWXA652DgTmR8SCiHgPmASMqnNN1o6IeAhYWe86rDoccLWzM/BKyedFaZuZdRAHnJkVlgOudl4F+pZ83iVtM7MO4oCrnceB/pJ2l7Q5cDwwpc41mW1SHHA1EhFrgbOB+4BngckR8XR9q7L2SLoV+COwl6RFkk6vd01WOV+qZWaF5RGcmRWWA87MCssBZ2aF5YAzs8JywJlZYTngckTSPyXNlTRP0u2SttqIvm6SdEz6/vq2bgQgaZikQyrYxkuSPvT0pdbam63zVsZtXSLpvKw1WrE54PJlTUTsFxEDgfeAM0sXSqroObcRcUZEPNPGKsOAzAFnVm8OuPx6GNgzHV09LGkK8IykzpJ+IulxSU9J+gqAEv8nvT/d/wN2bOpI0kxJg9P3wyU9IelJSTMk7UYSpN9MR4+HS9pB0u/SbTwu6dD0e3tKul/S05Kup4znn0v6g6Q56feMbbbsZ2n7DEk7pG17SJqefs/Dkvauym/TCslPts+hdKQ2ApieNu0PDIyIF9OQWBURn5TUFXhE0v3AJ4C9SO5N1xt4BrixWb87AL8GhqZ99YiIlZKuA96KiCvT9X4L/CwiZknqR3K1xseAi4FZEfF9SUcB5VwF8OV0G1sCj0v6XUSsALYGZkfENyVdlPZ9NsnDYM6MiBckHQRcCxxRwa/RNgEOuHzZUtLc9P3DwA0kU8c/RcSLaftngH2b9q8B2wH9gaHArRHxT+A1Sf/ZQv9DgIea+oqI1u6L9q/AAGn9AK2bpG3SbfyP9HvvkfRGGT/TOZK+mL7vm9a6AlgH3Ja2TwR+n27jEOD2km13LWMbtolywOXLmojYr7Qh/Yf+dmkT8PWIuK/Zep+rYh2dgCER8U4LtZRN0jCSsDw4IlZLmgls0crqkW73781/B2at8T644rkP+KqkzQAkfVTS1sBDwHHpPro+wKdb+N5HgaGSdk+/t0fa/iawbcl69wNfb/ogab/07UPAiWnbCKB7O7VuB7yRhtveJCPIJp2AplHoiSRT338AL0r6UroNSRrUzjZsE+aAK57rSfavPZE+OOX/kozU7wReSJfdTHLHjA1ExDJgLMl08Ek+mCJOBb7YdJABOAcYnB7EeIYPjuZeShKQT5NMVV9up9bpQBdJzwJXkARsk7eBA9Of4Qjg+2n7aOD0tL6n8W3grQ2+m4iZFZZHcGZWWA44MyssB5yZFZYDzswKywFnZoXlgDOzwnLAmVlh/X/bRRccPqtd8QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lr = LogisticRegression(C =best_c, penalty='l1',solver='liblinear')\n",
    "lr.fit(X_train_under_sample, Y_train_under_sample.values.ravel())\n",
    "Y_pred_under_sample = lr.predict(X_test_under_sample.values)\n",
    "\n",
    "# compute confusion_matrix\n",
    "cnf_matrix = confusion_matrix(Y_test_under_sample, Y_pred_under_sample)\n",
    "np.set_printoptions(precision=2)\n",
    "print('Recall metric in test: ', cnf_matrix[1,1] / (cnf_matrix[1,0]+cnf_matrix[1,1]))\n",
    "\n",
    "#plot\n",
    "class_names = [0,1]\n",
    "plt.figure()\n",
    "plot_confusion_matrix(cnf_matrix, labels=class_names)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "lr = LogisticRegression(C =best_c, penalty='l1',solver='liblinear')\n",
    "lr.fit(X_train_under_sample, Y_train_under_sample.values.ravel())\n",
    "Y_pred  = lr.predict(X_test.values) # 将测试集改成所有的样本\n",
    "\n",
    "# compute confusion_matrix\n",
    "cnf_matrix = confusion_matrix(Y_test, Y_pred)\n",
    "np.set_printoptions(precision=2)\n",
    "print('Recall metric in test: ', cnf_matrix[1,1] / (cnf_matrix[1,0]+cnf_matrix[1,1]))\n",
    "\n",
    "#plot\n",
    "class_names = [0,1]\n",
    "plt.figure()\n",
    "plot_confusion_matrix(cnf_matrix, labels=class_names)\n",
    "plt.show()\n",
    "# 会发现错误率很大"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "best_c = printing_kfold_score(X_train, Y_train)\n",
    "best_c # 全样本训练,发现 recall 很低"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "lr = LogisticRegression(C = 0.01, penalty='l1',solver='liblinear')\n",
    "lr.fit(X_train_under_sample, Y_train_under_sample.values.ravel())\n",
    "Y_pred_unser_sample_pro  = lr.predict_proba(X_test_under_sample.values) # 预测一个概率值\n",
    "\n",
    "thresholds = np.arange(0.1,1,0.1) # [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]\n",
    "plt.figure(figsize=(8,6))\n",
    "j = 1\n",
    "for i in thresholds:\n",
    "  y_test_pred_high_recall = Y_pred_unser_sample_pro[:,1] > i\n",
    "  plt.subplot(3,3,j)\n",
    "  j += 1\n",
    "  cnf_matrix = confusion_matrix(Y_test_under_sample,y_test_pred_high_recall)\n",
    "  np.set_printoptions(precision=2)\n",
    "  print('Recall metric in test: ', cnf_matrix[1,1] / (cnf_matrix[1,0]+cnf_matrix[1,1]))\n",
    "  class_names = [0,1]\n",
    "  plot_confusion_matrix(cnf_matrix, labels=class_names, title='Threshold >= %s' % i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SMOTE 算法\n",
    "\n",
    "1. 对少数类中每个样本x， 以欧式距离为标准计算它到少数类样本集中所有样本的距离，得到其K近邻\n",
    "2.\n",
    "3.\n",
    "`pip install imblearn`  imbalanced-learn-0.7.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.ensemble import  RandomForestClassifier\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import  train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "credit_card_data = pd.read_csv('data/creditcard.csv')\n",
    "columns = credit_card_data.columns\n",
    "\n",
    "feature_columns = columns.delete(len(columns) -1) # 将class 移到\n",
    "features = credit_card_data[feature_columns]\n",
    "labels = credit_card_data['Class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "features_train,features_test, labels_train, labels_test = train_test_split(features, labels,test_size=0.2,random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "over_sample = SMOTE(random_state=0)\n",
    "os_features,os_labels = over_sample.fit_sample(features_train, labels_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "227454"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(os_labels[os_labels==1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------\n",
      "c_param: 0.01\n",
      "------------------------------\n",
      "\n",
      "iteration:  1 ,recall_score: 0.8903225806451613\n",
      "iteration:  2 ,recall_score: 0.8947368421052632\n",
      "iteration:  3 ,recall_score: 0.9688834790306518\n",
      "iteration:  4 ,recall_score: 0.9578043767380002\n",
      "iteration:  5 ,recall_score: 0.958408898561238\n",
      "\n",
      "Mean recall score 0.9340312354160629\n",
      "\n",
      "------------------------------\n",
      "c_param: 0.1\n",
      "------------------------------\n",
      "\n",
      "iteration:  1 ,recall_score: 0.8903225806451613\n",
      "iteration:  2 ,recall_score: 0.8947368421052632\n",
      "iteration:  3 ,recall_score: 0.9702998782781896\n",
      "iteration:  4 ,recall_score: 0.9600246205251646\n",
      "iteration:  5 ,recall_score: 0.9602554379485827\n",
      "\n",
      "Mean recall score 0.9351278719004721\n",
      "\n",
      "------------------------------\n",
      "c_param: 1\n",
      "------------------------------\n",
      "\n",
      "iteration:  1 ,recall_score: 0.8903225806451613\n",
      "iteration:  2 ,recall_score: 0.8947368421052632\n",
      "iteration:  3 ,recall_score: 0.9705211906606175\n",
      "iteration:  4 ,recall_score: 0.960321385783845\n",
      "iteration:  5 ,recall_score: 0.9608489684659435\n",
      "\n",
      "Mean recall score 0.9353501935321662\n",
      "\n",
      "------------------------------\n",
      "c_param: 100\n",
      "------------------------------\n",
      "\n",
      "iteration:  1 ,recall_score: 0.8903225806451613\n",
      "iteration:  2 ,recall_score: 0.8947368421052632\n",
      "iteration:  3 ,recall_score: 0.9703884032311608\n",
      "iteration:  4 ,recall_score: 0.9601675075015662\n",
      "iteration:  5 ,recall_score: 0.9607940119365582\n",
      "\n",
      "Mean recall score 0.9352818690839418\n",
      "\n",
      "------------------------------\n",
      "c_param: 100\n",
      "------------------------------\n",
      "\n",
      "iteration:  1 ,recall_score: 0.8903225806451613\n",
      "iteration:  2 ,recall_score: 0.8947368421052632\n",
      "iteration:  3 ,recall_score: 0.9704105344694036\n",
      "iteration:  4 ,recall_score: 0.9603653510073532\n",
      "iteration:  5 ,recall_score: 0.960761038018927\n",
      "\n",
      "Mean recall score 0.9353192692492216\n",
      "\n",
      "******************************\n",
      "best_c: 1.0\n",
      "******************************\n"
     ]
    }
   ],
   "source": [
    "os_features = pd.DataFrame(os_features)\n",
    "os_labels = pd.DataFrame(os_labels)\n",
    "best_c = printing_kfold_score(os_features,os_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'LogisticRegression' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-322d2a03df4d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLogisticRegression\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mC\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0mbest_c\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpenalty\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'l1'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msolver\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'liblinear'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mos_labels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0my_pred\u001b[0m  \u001b[0;34m=\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures_test\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# 将测试集改成所有的样本\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# compute confusion_matrix\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'LogisticRegression' is not defined"
     ]
    }
   ],
   "source": [
    "lr = LogisticRegression(C =best_c, penalty='l1',solver='liblinear')\n",
    "lr.fit(os_features, os_labels.values.ravel())\n",
    "y_pred  = lr.predict(features_test.values) # 将测试集改成所有的样本\n",
    "\n",
    "# compute confusion_matrix\n",
    "cnf_matrix = confusion_matrix(labels_test, y_pred)\n",
    "np.set_printoptions(precision=2)\n",
    "print('Recall metric in test: ', cnf_matrix[1,1] / (cnf_matrix[1,0]+cnf_matrix[1,1]))\n",
    "\n",
    "#plot\n",
    "class_names = [0,1]\n",
    "plt.figure()\n",
    "plot_confusion_matrix(cnf_matrix, labels=class_names)\n",
    "plt.show()\n",
    "# 会发现错误率很大\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall metric in test:  0.9115646258503401\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUUAAAEWCAYAAADxboUEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAliklEQVR4nO3debwWZf3/8df7gAsubLKIIGqFGF9LVL6KWoZLLFpBfV0wUjKSzN3q69JGufTNNgvXLElQcyk3MgUJJbVfKqC4oQapyKaAIOZSin5+f8x1w5zjOfe5bzjrfd5PH/M4M9dcc8015+jHa+aauS5FBGZmlqlq7gqYmbUkDopmZjkOimZmOQ6KZmY5DopmZjkOimZmOQ6KFUZSB0l/krRW0h82oZwxku5pyLo1B0l3Sxrb3PWw1sNBsZlI+qKkOZLekLQ8/cf7iQYo+gigJ7BdRBy5sYVExPURMbQB6lONpCGSQtJtNdL3SOmzSiznB5Kuqy9fRIyIiMkbWV1rgxwUm4GkbwC/BH5EFsD6ApcDIxug+J2Af0TEugYoq7GsBPaTtF0ubSzwj4Y6gTL+99vKFxFemnABOgFvAEcWybMFWdBclpZfAlukfUOAJcA3gRXAcuD4tO+HwDvAu+kc44AfANflyt4ZCKB92v4y8DzwL+AFYEwu/cHccfsDs4G16ef+uX2zgPOBv6Vy7gG61XFthfpfCZyc0toBS4HvA7NyeX8FLAZeB+YCn0zpw2tc5+O5elyY6vE28JGU9tW0/wrgllz5FwEzATX3vxdeWs7i/5M2vf2ALYHbiuT5DjAYGAjsAewDfDe3f3uy4NqbLPBdJqlLREwga33eFBHbRMTVxSoiaWtgIjAiIrYlC3zzasnXFfhzyrsd8AvgzzVael8Ejgd6AJsD3yp2bmAKcFxaHwY8RfY/gLzZZL+DrsDvgT9I2jIiptW4zj1yxxwLjAe2BRbVKO+bwMckfVnSJ8l+d2Mjwt+62noOik1vO2BVFL+9HQOcFxErImIlWQvw2Nz+d9P+dyPiLrLWUv+NrM/7wO6SOkTE8oh4upY8hwMLIuLaiFgXETcAzwKfzeX5XUT8IyLeBm4mC2Z1ioj/B3SV1J8sOE6pJc91EfFqOufPyVrQ9V3nNRHxdDrm3RrlvUX2e/wFcB1wakQsqac8a2McFJveq0A3Se2L5NmB6q2cRSltfRk1gupbwDblViQi3gSOBk4Elkv6s6TdSqhPoU69c9svb0R9rgVOAQ6ilpazpG9Jeib1pL9G1jruVk+Zi4vtjIiHyR4XiCx4m1XjoNj0/g78BxhVJM8ysg6Tgr588NayVG8CW+W2t8/vjIjpEfFpoBdZ6+83JdSnUKelG1mngmuBk4C7UituvXR7exZwFNAlIjqTPc9Uoep1lFn0VljSyWQtzmWpfLNqHBSbWESsJetQuEzSKElbSdpM0ghJP0nZbgC+K6m7pG4pf72vn9RhHnCgpL6SOgHnFnZI6ilpZHq2+B+y2/D3aynjLmDX9BpRe0lHAwOAOzeyTgBExAvAp8ieoda0LbCOrKe6vaTvAx1z+18Bdi6nh1nSrsAFwJfIbqPPkjRw42pvlcpBsRmk52PfIOs8WUl2y3cKcHvKcgEwB3gCeBJ4NKVtzLlmADelsuZSPZBVpXosA1aTBaiv11LGq8BnyDoqXiVrYX0mIlZtTJ1qlP1gRNTWCp4OTCN7TWcR8G+q3xoXXkx/VdKj9Z0nPa64DrgoIh6PiAXAt4FrJW2xKddglUXueDMz28AtRTOzHAdFM7McB0UzsxwHRTOznGIvEDc5te8Q2nzb5q6GleG/du3T3FWwMixd/BKrX12l+nPWrV3HnSLWvV1S3nh75fSIGL4p52tqLSsobr4tW/Q/qrmrYWW4456fNncVrAwjP33AJpcR694u+b/Tf8+7rL4vkFqcFhUUzaw1EFTwqGwOimZWHgFV7Zq7Fo3GQdHMyqdNeizZojkomlmZfPtsZladW4pmZolwS9HMbAO5pWhmVo17n83MCtzRYma2gajo2+fKDfdm1nhUVdpSrAipv6R5ueV1SWdI6ipphqQF6WeXlF+SJkpaKOkJSXvlyhqb8i+QNDaXvrekJ9MxE6X6o7mDopmVSQ0SFCPiuYgYGBEDgb3JZoG8DTgHmBkR/YCZaRtgBNAvLeOBK2D9vOQTgH3J5kifUAikKc8JuePqHZzCQdHMyiOgXbvSltIdAvwzIhYBI4HJKX0yG2a+HAlMicxDQGdJvYBhwIyIWB0Ra4AZwPC0r2NEPBTZvCtTKD6LJuBnima2MRr+meJoslksAXpGxPK0/jLQM633pvrkZUtSWrH0JbWkF+WWopmVqazb526S5uSW8R8oTdoc+BwbZmhcL7XwmnR2PbcUzax8pbcUV0XEoHryjAAejYhX0vYrknpFxPJ0C7wipS8Fdswd1yelLQWG1EifldL71JK/KLcUzax8DdDRknMMG26dAaYChR7kscAdufTjUi/0YGBtus2eDgyV1CV1sAwFpqd9r0sanHqdj8uVVSe3FM2sPGq4z/wkbQ18GvhaLvnHwM2SxgGLgMIw33cBhwELyXqqjweIiNWSzgdmp3znRcTqtH4ScA3QAbg7LUU5KJpZ+RroM7+IeBPYrkbaq2S90TXzBnByHeVMAibVkj4H2L2cOjkomlmZ/JmfmVl1FfyZn4OimZXH4ymameX59tnMrDqPp2hmluNnimZmiXz7bGZWnVuKZmYblDBWa6vloGhmZclmI3BQNDPLSKjKQdHMbD23FM3MchwUzcxyHBTNzAqUlgrloGhmZRFyS9HMLK+qyl+0mJmt55aimVmBnymamVVXyS3Fyn0wYGaNotDRUspSb1lSZ0l/lPSspGck7Sepq6QZkhakn11SXkmaKGmhpCck7ZUrZ2zKv0DS2Fz63pKeTMdMVAmVclA0s7KpSiUtJfgVMC0idgP2AJ4BzgFmRkQ/YGbaBhgB9EvLeOAKAEldgQnAvsA+wIRCIE15TsgdN7y+Cjkomll5RIO0FCV1Ag4ErgaIiHci4jVgJDA5ZZsMjErrI4EpkXkI6CypFzAMmBERqyNiDTADGJ72dYyIh9L0qFNyZdXJQdHMytZAt8+7ACuB30l6TNJvJW0N9IyI5SnPy0DPtN4bWJw7fklKK5a+pJb0ohwUzaxsZQTFbpLm5JbxuWLaA3sBV0TEnsCbbLhVBiC18KKprqtQKTOzkpX5RcuqiBhUx74lwJKIeDht/5EsKL4iqVdELE+3wCvS/qXAjrnj+6S0pcCQGumzUnqfWvIX5ZaimZVPJS5FRMTLwGJJ/VPSIcB8YCpQ6EEeC9yR1qcCx6Ve6MHA2nSbPR0YKqlL6mAZCkxP+16XNDj1Oh+XK6tObimWqN9OPbj2oq+s396l93acf8Wf6bTtVnzlC/uzcs0bAEy4dCrTH5zPwfvuxvmnfY7NN2vPO++u49u/vJ2/zv4HAEcM3Yuzxg2jXbsq7r7/Kb47Mfs7nfalg/ny5/dj3br3WbXmDU784XW8tHxN019shTj79K9x74xpbNetO9PunwPAa2tWc9oJx7Fk8SL67LgTl/z2Wjp17kJEcN53vsWsv0ynQ4et+Mklv2b3j+8JwC03XsdlF18EwMlnns3/jP5StfOMP/YIXlr04vpzVDw16Gd+pwLXS9oceB44nqyxdrOkccAi4KiU9y7gMGAh8FbKS0SslnQ+MDvlOy8iVqf1k4BrgA7A3WkpykGxRAsWrWDw6B8DUFUl/jn9Qqbe9zjHfm4/LrnuPn557cxq+V997Q2OOOPXLF+5lgEf7sWfLj+ZDw/7Ll07bc2PzhjF/mN+wqo1b/Cb845lyD67MuuRfzDv2cUcMOYB3v73u5xw5Ce48PRRHHvO75rjcivC/4w+lmPHnci3TjlhfdqVE3/O/gcO4cTTvsWVE3/GlRN/ztnfv4BZM6fz4vMLuffhJ5k3dzbfP+t0bp12P6+tWc0lP/sRt894EEmMPPQADh1+OJ06Z298TL/zdrbaepvmusRm01Avb0fEPKC22+tDaskbwMl1lDMJmFRL+hxg93Lq5NvnjXDQPv15YcnKoq24x59bwvKVawGY/8/lbLnFZmy+WXt26b0dC19ayarUsrz34WcZdchAAO6fs4C3//0uAI888SK9e3Zu1OuodPvs9wk6d+5aLe0v0+7kC0ePAeALR49hxt1/ytLvvpPPHzUGSew5aB9eX7uWFa8s5/77/sIBnzqYzl260qlzFw741MH89d4ZALz5xhtcfeUlnHzm2U17YS1BA9w+t1QOihvhyGF7c/O0ueu3Txx9II/cdC5XThhD5207fCD/5w8dyLxnF/POu+v45+KV7LpzD/r26kq7dlV87qA96NOzyweO+fKo/Zj+t/mNeh1t0aqVK+jRsxcA3Xtsz6qV2TP8V15exg47bHgmv/0OvXl5+TJeWb6MXr2rp7+yfBkAF190HuO+fhodOmzVhFfQMjTUFy0tUaMGRUnDJT2XPrE5p/4jWr7N2rfj8E99jFtnPAbAb/7wAAM++wP2Hf1jXl71Oj/+xheq5f/oh7bngtNGcsoFNwLw2r/e5rQf3cR1F32FmZPOZNGyV3n//ferHTP6sP9mrwF9uXhy9Vtya1ib8h/u/Ccf56UXn2fY4SMbuFYtX6kB0UGxBkntgMvIPs0ZABwjaUBjna+pDPvEAOY9u5gVq/8FwIrV/+L994OIYNKtf2PQ7jutz9u7R2du+sV4vvq9a3lhyar16Xfd/xQHHvczhoz9Of94cQULFq1Yv++gfftz9rhhHHHGr3nn3XVNd2FtRLfuPVjxSvZe8IpXlrNdt+4A9Nx+B5Yt2/Ce78vLlrJ9rx3o2WsHli+tnt6z1w48Nudhnpz3KAfuvRtHf/YQXvznAr44aljTXkwzclDcOPsACyPi+Yh4B7iR7DOdVu2o4YOq3Tpv363j+vWRB+/B/H9m/8F12qYDt15yIt+beAd/f/z5amV075I9mO+8bQfGH/VJfnfb3wHYo38fLv3OaI4489fre7OtYR0y7HBuvel6AG696XoOHf4ZAA4dfji33Xw9EcFjcx5h244d6dGzFwcedCgP/nUma19bw9rX1vDgX2dy4EGHMub48fz9yee5f+6z3PSnmez84X78/vbpzXlpTaoBv31ucRqz97m2T2/2rZkpveGeveW+Wcvuxdtqy805eN/dOOWCG9anXXj6KD7evw8RwaLlqzk17Ttx9IF8eMfunDt+BOeOHwHAZ79+KSvXvMHPzjqCj+2afW30f1dNY+FLWUvxR2eOYuuttuD6n4wDYPHLazjyjF835SVWlNO/NpaH/3Y/a1a/ygF7fITTz/ouJ572TU494Vhuvn4yvfv05ZLfXgvAkEOHM+sv0zl4n93ZcqutuOhXVwLQuUtXTvnGOYwa+kkATv3muXTu0rXOc7YVrbUVWAplvdyNULB0BDA8Ir6ato8F9o2IU+o6pmqrHrFF/6Pq2m0t0NP3/LS5q2BlGPnpA3hy3qObFNG22L5f9BkzsaS8z//isLlFvmhpkRqzpVjXJzlm1ooJqOCGYqM+U5wN9JO0S3pbfTTZZzpm1qpVdu9zo7UUI2KdpFPIvktsB0yKiKcb63xm1nSqWmknSika9TO/iLiL7HtFM6sUquzbZ3/7bGZlEW4pmplV45aimVlOa+1EKYWDopmVx88Uzcw2EGrIQWZbHAdFMyubW4pmZjl+pmhmVuBnimZmG2TfPlduVKzcp6Vm1mik0pb6y9GLkp6UNE/SnJTWVdIMSQvSzy4pXZImppH8n5C0V66csSn/Akljc+l7p/IXpmPrrZWDopmVrapKJS0lOigiBuaGGDsHmBkR/YCZaRuyUfz7pWU8cAVkQRSYQDZe6z7AhEIgTXlOyB03vN5rK7XWZmZAeqbYqKPkjAQmp/XJwKhc+pTIPAR0ltQLGAbMiIjVEbEGmAEMT/s6RsRDaXrUKbmy6uSgaGZlKYynWOLtczdJc3LL+BrFBXCPpLm5fT0jYnlafxnomdZrG82/dz3pS2pJL8odLWZWprJagavqGXn7ExGxVFIPYIakZ/M7IyIkNc70AHVwS9HMytZQHS0RsTT9XAHcRvZM8JV060v6WZjusq7R/Iul96klvSgHRTMrjxqmo0XS1pK2LawDQ4GnyEboL/QgjwXuSOtTgeNSL/RgYG26zZ4ODJXUJXWwDAWmp32vSxqcep2Py5VVJ98+m1lZGvA9xZ7Abams9sDvI2KapNnAzZLGAYuAwmx2dwGHAQuBt4DjASJitaTzyaZAATgvIlan9ZOAa4AOwN1pKcpB0czK1hBBMSKeB/aoJf1V4JBa0gM4uY6yJgGTakmfA+xeTr0cFM2sbBX8QYuDopmVr5I/83NQNLPyeEAIM7MNskFmKzcqOiiaWdmqKrip6KBoZmWr4JjooGhm5ZHc0WJmVk0FP1KsOyhKuoRsBItaRcRpjVIjM2vx2mpHy5wmq4WZtRoi64GuVHUGxYiYnN+WtFVEvNX4VTKzlq6CG4r1j5IjaT9J84Fn0/Yeki5v9JqZWctU4qjbrbUzppShw35JNtz3qwAR8ThwYCPWycxauIYaT7ElKqn3OSIW14j67zVOdcyspRN+eXuxpP2BkLQZcDrwTONWy8xaskrufS7l9vlEsjHMegPLgIHUMaaZmVW+Um+dW2tjst6WYkSsAsY0QV3MrJWo5NvnUnqfPyTpT5JWSloh6Q5JH2qKyplZy6QSl9aolNvn3wM3A72AHYA/ADc0ZqXMrGVr66/kbBUR10bEurRcB2zZ2BUzs5Yp630ubWmN6gyKkrpK6grcLekcSTtL2knSWWSzaplZW6TSpjcttYdaUjtJj0m6M23vIulhSQsl3SRp85S+RdpemPbvnCvj3JT+nKRhufThKW2hpHNKqU+xjpa5ZANCFK7sa7l9AZxb0hWbWcVp4Fvjwmt+HdP2RcDFEXGjpCuBccAV6eeaiPiIpNEp39GSBgCjgf8ie8T3F0m7prIuAz4NLAFmS5oaEfOLVabOlmJE7BIRH0o/ay7uaDFroxry9llSH+Bw4LdpW8DBwB9TlsnAqLQ+Mm2T9h+S8o8EboyI/0TEC2TzQu+TloUR8XxEvAPcmPIWVdIXLZJ2BwaQe5YYEVNKOdbMKk8ZLcVukvIjbl0VEVfltn8JnAVsm7a3A16LiHVpewnZO9Kkn4sBImKdpLUpf2/goVyZ+WMW10jft74K1xsUJU0AhpAFxbuAEcCDgIOiWRtVxs3zqogYVGsZ0meAFRExV9KQBqlYAyilpXgEsAfwWEQcL6kncF3jVsvMWioJ2jVM1/IBwOckHUZ2F9oR+BXQWVL71FrsAyxN+ZcCOwJLJLUHOpENVFNIL8gfU1d6nUp5JeftiHgfWCepI7CixonMrI1piPcUI+LciOgTETuTdZTcGxFjgPvIGmMAY4E70vrUtE3af29EREofnXqndwH6AY8As4F+qTd783SOqfVdWyktxTmSOgO/IeuRfgP4ewnHmVmFauT3ss8GbpR0AfAYcHVKvxq4VtJCYDVZkCMinpZ0MzAfWAecHBHvZfXUKcB0oB0wKSKeru/kpXz7fFJavVLSNKBjRDxRxgWaWQURavBvnyNiFjArrT9P1nNcM8+/gSPrOP5C4MJa0u+izPeqi01ctVexfRHxaDknMrMK0YpHwClFsZbiz4vsC7J3iRrUnh/ty98evrShizWzZPN2pXQj1K+1ftdcimITVx3UlBUxs9ZBQLu2GBTNzOrSWgd7KIWDopmVzUHRzCzJphqo3KhYysjbkvQlSd9P230lfaC73MzajjY5nmLO5cB+wDFp+19kw/GYWRvVpieuAvaNiL0kPQYQEWsKgz6aWdsjoH1rjXglKCUoviupHdm7iUjqDrzfqLUysxatgmNiSUFxInAb0EPShWQfYn+3UWtlZi2W1PCf+bUkpXz7fL2kucAhZC3nURHxTKPXzMxarAqOiSUNMtsXeAv4Uz4tIl5qzIqZWcvVWnuWS1HK7fOf2TCB1ZbALsBzZJPEmFkbIxpskNkWqZTb54/lt9PoOSfVkd3MKl0rfgexFGV/0RIRj0qqd/IXM6tcKmeWllamlGeK38htVgF7AcsarUZm1qIVpjitVKW0FLfNra8je8Z4S+NUx8xagzYbFNNL29tGxLeaqD5m1gpU8oAQxaYjaJ8mnD6gKStkZi1bNsVpc9ei8RS7tEfSz3mSpko6VtIXCktTVM7MWqaq9FVLfUsxkraU9IikxyU9LemHKX0XSQ9LWijppsJYC2kK05tS+sOSds6VdW5Kf07SsFz68JS2UNI5JV1bCXm2JJtw+mDgM8Bn008za4MKHS0NMHTYf4CDI2IPYCAwXNJg4CLg4oj4CLAGGJfyjwPWpPSLUz4kDSCb7vS/gOHA5ZLapcd/lwEjgAHAMSlvUcWeKfZIPc9PseHl7YKo93LNrGI1xCPFNJH9G2lzs7QUJsX7YkqfDPwAuAIYmdYB/ghcquzh5kjgxoj4D/BCmhe6MObrwjRlKpJuTHnnF6tXsaDYDtgGan0hyUHRrM0SVaW/p9hN0pzc9lURcdX6krLW3FzgI2Stun8Cr0XEupRlCdA7rfcGFgOk/o61wHYp/aHcOfLHLK6RXu871sWC4vKIOK++AsysbRFltRRXRcSgunZGxHvAQEmdyUbj2m1T67epigXFyu1zN7ONJ2jfwC8qRsRrku4jG+W/c+HtF6APsDRlWwrsCCyR1B7oRNbfUUgvyB9TV3qdinW0HFLCtZhZG1NoKW7qdASSuqcWIpI6AJ8GngHuIxu3FWAscEdan5q2SfvvTc8lpwKjU+/0LkA/srdnZgP9Um/25mSdMVPru746W4oRsbq+g82sbWqgQWZ7AZPTc8Uq4OaIuFPSfOBGSRcAjwFXp/xXA9emjpTVZEGOiHha0s1kHSjrgJPTbTmSTgGmk/WRTIqIp+urlKc4NbOyNVDv8xPAnrWkP8+G3uN8+r+BI+so60LgwlrS7wLuKqdeDopmVhZR2gvOrZWDopmVRw12+9wiOSiaWVmyL1ocFM3M1qvckOigaGYboYIbig6KZlYutc3xFM3MauPeZzOzGtzRYmZWoDY6HYGZWW18+2xmVoNbimZmOZUbEh0UzaxMAtq5pWhmtkEFx0QHRTMrl1AF30A7KJpZ2dxSNDNLsldyKjcqOiiaWXlKmH+lNXNQNLOyVfJnfpX8Ynqz+NpXv0LfHXqw98Dd16f9cML3+O89P86+ew/kMyOGsmzZsmasoW3M32jO7Nlss2V7br3lj01d3RYnG2S2tKU1clBsYMeO/TJ33DmtWtqZ3/xfZj/2BA/PnceIwz7D/11wXjPVzqD8v9F7773Hd799Nod+emhTV7XFUon/FC1D2lHSfZLmS3pa0ukpvaukGZIWpJ9dUrokTZS0UNITkvbKlTU25V8gaWwufW9JT6ZjJqqET3EcFBvYJz55IF27dq2W1rFjx/Xrb731ZkV/ItUalPs3uvzSSxj1+f+he/ceTVbHlq4h5n0mm470mxExABgMnCxpAHAOMDMi+gEz0zbACLI5nfsB44ErsrqoKzAB2JdsFsAJhUCa8pyQO254fZXyM8UmMuF73+H666bQqVMnps24r7mrY7Wo7W+0dOlSpt5xG9P/ch9f++rsZq5hy9EQ7ylGxHJgeVr/l6RngN7ASGBIyjYZmAWcndKnREQAD0nqLKlXyjujMFe9pBnAcEmzgI4R8VBKnwKMAu4uVq9GaylKmiRphaSnGuscrckPz7+QhS8sZvQxY7jy8kubuzpWi9r+Rv/7zTO44EcXUVXlm6qCMp8pdpM0J7eMr7VMaWeyOaAfBnqmgAnwMtAzrfcGFucOW5LSiqUvqSW9qMb8S19DCU3VtuboY8Zw+223NHc1rIj83+jRuXM47kuj6f+Rnbnt1j9yxqknMfWO25u3gs1NoqrEBVgVEYNyy1UfLE7bALcAZ0TE6/l9qVUYTXJdSaMFxYi4H1jdWOW3JgsXLFi/fufUO9i1/27NWBurTV1/o2cXvMBzC1/kuYUv8vkvHMEvL7mcz40c1Uy1bDlU4lJvOdJmZAHx+oi4NSW/km6LST9XpPSlwI65w/uktGLpfWpJL6rZnymm5vR4gB379m3m2my64750DA/8dRarVq3iwzv34Xvf/yHTpt3Fgn88R5Wq6LvTTky87Mrmrmab5r/RpmmoeZ9TT/DVwDMR8YvcrqnAWODH6ecdufRTJN1I1qmyNiKWS5oO/CjXuTIUODciVkt6XdJgstvy44BL6q1X1jptHOk5wZ0RsXt9eQH23ntQ/O3hOY1WH7O27oB9BzF37pxNimgf/die8bvbSuss3K9fl7kRMai2fZI+ATwAPAm8n5K/TRbAbgb6AouAo1KAE3Ap2WO5t4DjI2JOKusr6ViACyPidyl9ENmjvA5kHSynRj1Br9lbimbWCjXAW2UR8WCRkg6pJX8AJ9dR1iRgUi3pc4CSGmUFDopmVjZ/5rcRJN0A/B3oL2mJpHGNdS4za1oN1dHSEjVaSzEijmmsss2smbXWiFcC3z6bWVmyVmDlRkUHRTMrj8dTNDOrroJjooOimZVLFT3Sk4OimZWtgmOig6KZlac1v25TCgdFMytfBUdFB0UzK5tfyTEzy/EzRTOzAr+naGZWnW+fzcwS4ZaimVk1FRwTHRTNbCNUcFR0UDSzslXyILMOimZWtsoNiQ6KZrYxKjgqNtp0BGZWmQqDzJbyT71lSZMkrZD0VC6tq6QZkhakn11SuiRNlLRQ0hOS9sodMzblXyBpbC59b0lPpmMmqoThfRwUzaw86eXtUpYSXEM2ZWneOcDMiOgHzEzbACOAfmkZD1wBWRAFJpDNBb0PMCE3B/QVwAm542qe6wMcFM2sbA01cVVE3A+srpE8Epic1icDo3LpUyLzENBZUi9gGDAjIlZHxBpgBjA87esYEQ+l6VGn5Mqqk58pmlmZyhpktpukObntqyLiqnqO6RkRy9P6y0DPtN4bWJzLtySlFUtfUkt6UQ6KZla2Mt7IWRURgzb2PBERkmJjj98Yvn02s7KUeuu8CR3Ur6RbX9LPFSl9KbBjLl+flFYsvU8t6UU5KJpZ+Ro3Kk4FCj3IY4E7cunHpV7owcDadJs9HRgqqUvqYBkKTE/7Xpc0OPU6H5crq06+fTazsjXUKDmSbgCGkD17XELWi/xj4GZJ44BFwFEp+13AYcBC4C3geICIWC3pfGB2yndeRBQ6b04i6+HuANydlqIcFM2sbA31lV9EHFPHrkNqyRvAyXWUMwmYVEv6HGD3curkoGhm5RFUVfAXLQ6KZrYRKjcqOiiaWVk8yKyZWQ0VHBMdFM2sfG4pmpnllPGZX6vjoGhmZavckOigaGZlKmNYsFbJQdHMyuZ5n83M8io3Jjoomln5KjgmOiiaWbnkKU7NzAoq/YsWj6doZpbjlqKZla2SW4oOimZWNr+SY2ZW4Je3zcw2qPSOFgdFMyubb5/NzHLcUjQzy6ngmOigaGYboYKjooOimZVFUNGf+SmbSrVlkLSSbPLrStMNWNXclbCyVOrfbKeI6L4pBUiaRvb7KcWqiBi+Kedrai0qKFYqSXMiYlBz18NK579Z2+Vvn83MchwUzcxyHBSbxlXNXQErm/9mbZSfKZqZ5bilaGaW46BoZpbjoNiIJA2X9JykhZLOae76WP0kTZK0QtJTzV0Xax4Oio1EUjvgMmAEMAA4RtKA5q2VleAaoFW9bGwNy0Gx8ewDLIyI5yPiHeBGYGQz18nqERH3A6ubux7WfBwUG09vYHFue0lKM7MWzEHRzCzHQbHxLAV2zG33SWlm1oI5KDae2UA/SbtI2hwYDUxt5jqZWT0cFBtJRKwDTgGmA88AN0fE081bK6uPpBuAvwP9JS2RNK6562RNy5/5mZnluKVoZpbjoGhmluOgaGaW46BoZpbjoGhmluOg2IpIek/SPElPSfqDpK02oaxrJB2R1n9bbLAKSUMk7b8R53hR0gdmfasrvUaeN8o81w8kfavcOprV5KDYurwdEQMjYnfgHeDE/E5JGzWPd0R8NSLmF8kyBCg7KJq1Rg6KrdcDwEdSK+4BSVOB+ZLaSfqppNmSnpD0NQBlLk3jO/4F6FEoSNIsSYPS+nBJj0p6XNJMSTuTBd8zUyv1k5K6S7olnWO2pAPSsdtJukfS05J+SzZvelGSbpc0Nx0zvsa+i1P6TEndU9qHJU1LxzwgabcG+W2aJRvVsrDmlVqEI4BpKWkvYPeIeCEFlrUR8d+StgD+JukeYE+gP9nYjj2B+cCkGuV2B34DHJjK6hoRqyVdCbwRET9L+X4PXBwRD0rqS/bVzkeBCcCDEXGepMOBUr4G+Uo6RwdgtqRbIuJVYGtgTkScKen7qexTyCaUOjEiFkjaF7gcOHgjfo1mtXJQbF06SJqX1h8Aria7rX0kIl5I6UOBjxeeFwKdgH7AgcANEfEesEzSvbWUPxi4v1BWRNQ1ruChwABpfUOwo6Rt0jm+kI79s6Q1JVzTaZI+n9Z3THV9FXgfuCmlXwfcms6xP/CH3Lm3KOEcZiVzUGxd3o6IgfmEFBzezCcBp0bE9Br5DmvAelQBgyPi37XUpWSShpAF2P0i4i1Js4At68ge6byv1fwdmDUkP1OsPNOBr0vaDEDSrpK2Bu4Hjk7PHHsBB9Vy7EPAgZJ2Scd2Ten/ArbN5bsHOLWwIWlgWr0f+GJKGwF0qaeunYA1KSDuRtZSLagCCq3dL5Ldlr8OvCDpyHQOSdqjnnOYlcVBsfL8lux54aNp8qVfk90R3AYsSPumkI0EU01ErATGk92qPs6G29c/AZ8vdLQApwGDUkfOfDb0gv+QLKg+TXYb/VI9dZ0GtJf0DPBjsqBc8CawT7qGg4HzUvoYYFyq39N4igdrYB4lx8wsxy1FM7McB0UzsxwHRTOzHAdFM7McB0UzsxwHRTOzHAdFM7Oc/w9uRRDq2u4iRAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lr = LogisticRegression(C = 0.01, penalty='l1',solver='liblinear')\n",
    "lr.fit(X_train_under_sample, Y_train_under_sample.values.ravel())\n",
    "Y_pred_unser_sample_pro  = lr.predict_proba(X_test_under_sample.values) # 预测一个概率值\n",
    "\n",
    "thresholds = np.arange(0.1,1,0.1) # [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]\n",
    "plt.figure(figsize=(8,6))\n",
    "j = 1\n",
    "for i in thresholds:\n",
    "  y_test_pred_high_recall = Y_pred_unser_sample_pro[:,1] > i\n",
    "  plt.subplot(3,3,j)\n",
    "  j += 1\n",
    "  cnf_matrix = confusion_matrix(Y_test_under_sample,y_test_pred_high_recall)\n",
    "  np.set_printoptions(precision=2)\n",
    "  print('Recall metric in test: ', cnf_matrix[1,1] / (cnf_matrix[1,0]+cnf_matrix[1,1]))\n",
    "  class_names = [0,1]\n",
    "  plot_confusion_matrix(cnf_matrix, labels=class_names, title='Threshold >= %s' % i)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------\n",
      "c_param: 0.01\n",
      "------------------------------\n",
      "\n",
      "iteration:  1 ,recall_score: 0.4925373134328358\n",
      "iteration:  2 ,recall_score: 0.6027397260273972\n",
      "iteration:  3 ,recall_score: 0.6833333333333333\n",
      "iteration:  4 ,recall_score: 0.5692307692307692\n",
      "iteration:  5 ,recall_score: 0.45\n",
      "\n",
      "Mean recall score 0.5595682284048672\n",
      "\n",
      "------------------------------\n",
      "c_param: 0.1\n",
      "------------------------------\n",
      "\n",
      "iteration:  1 ,recall_score: 0.5671641791044776\n",
      "iteration:  2 ,recall_score: 0.6164383561643836\n",
      "iteration:  3 ,recall_score: 0.6833333333333333\n",
      "iteration:  4 ,recall_score: 0.5846153846153846\n",
      "iteration:  5 ,recall_score: 0.525\n",
      "\n",
      "Mean recall score 0.5953102506435158\n",
      "\n",
      "------------------------------\n",
      "c_param: 1\n",
      "------------------------------\n",
      "\n",
      "iteration:  1 ,recall_score: 0.5522388059701493\n",
      "iteration:  2 ,recall_score: 0.6164383561643836\n",
      "iteration:  3 ,recall_score: 0.7166666666666667\n",
      "iteration:  4 ,recall_score: 0.6153846153846154\n",
      "iteration:  5 ,recall_score: 0.5625\n",
      "\n",
      "Mean recall score 0.612645688837163\n",
      "\n",
      "------------------------------\n",
      "c_param: 100\n",
      "------------------------------\n",
      "\n",
      "iteration:  1 ,recall_score: 0.5522388059701493\n",
      "iteration:  2 ,recall_score: 0.6164383561643836\n",
      "iteration:  3 ,recall_score: 0.7333333333333333\n",
      "iteration:  4 ,recall_score: 0.6153846153846154\n",
      "iteration:  5 ,recall_score: 0.575\n",
      "\n",
      "Mean recall score 0.6184790221704963\n",
      "\n",
      "------------------------------\n",
      "c_param: 100\n",
      "------------------------------\n",
      "\n",
      "iteration:  1 ,recall_score: 0.5522388059701493\n",
      "iteration:  2 ,recall_score: 0.6164383561643836\n",
      "iteration:  3 ,recall_score: 0.7333333333333333\n",
      "iteration:  4 ,recall_score: 0.6153846153846154\n",
      "iteration:  5 ,recall_score: 0.575\n",
      "\n",
      "Mean recall score 0.6184790221704963\n",
      "\n",
      "******************************\n",
      "best_c: 100.0\n",
      "******************************\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "100.0"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_c = printing_kfold_score(X_train, Y_train)\n",
    "best_c # 全样本训练,发现 recall 很低"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall metric in test:  1.0\n",
      "Recall metric in test:  1.0\n",
      "Recall metric in test:  1.0\n",
      "Recall metric in test:  0.9727891156462585\n",
      "Recall metric in test:  0.9319727891156463\n",
      "Recall metric in test:  0.8639455782312925\n",
      "Recall metric in test:  0.8231292517006803\n",
      "Recall metric in test:  0.7551020408163265\n",
      "Recall metric in test:  0.5782312925170068\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAAGDCAYAAADwNbC4AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABkgUlEQVR4nO3dd5gUVdbH8e+ZGXIQyTAEAVEEFFZRMSMmUFaMgCJixLi6uq5hXXNi1V3DGtH1FUEF3DUhCrqsWZQkBgyIBMkZJMMM5/2jarCn6QndM32rqvt8nqceuqurus4wv7pz61Z1tagqxhhjjDFhkRN0AcYYY4wxsaxzYowxxphQsc6JMcYYY0LFOifGGGOMCRXrnBhjjDEmVKxzYowxxphQsc5JGUTkdhEZ6WA754nIJymuW2qNIjJPRI5NvTrjkmUuHOz3YExwsr5zIiIbYqYdIrI55vnAoOuLOhGpJiLPicivIrJURK4tZdnOIjJBRFaKSMbegMcyl17lzZz//71FRApFRP1pm/0eKkeS+/4AEflRRNaJyHIRGS4idWNery8ir4nIRhGZLyJnx61/tj9/o4i8LiL1bd1wrRuz3HP+vrZnoteLZH3nRFVrF03AL8DvY+a9mMx7iUheeqpMHxFpJCKSxk3cDrQHWgNHA9eLSK8Slt0OjAEuTGM9gbPMhSNz/v//NUAPYD4wAPgGuNt+D5Xidsq/738KHKaquwFtgTzg7pjXHwe2AU2AgcCTItIJwP/3aWCQ//om4AlbNzzrFhGRw4F2lIeq2uRPwDzg2Lh5t+P9wXwBWA/MBLrFrXMD8DWwFW+n6g58BqwFvgJ6xCx/HjDHf6+5wMCY+Z8ADwJr/Nd6x6zXHHgTWA3MBi6Oq3FkzPNBeI3tKuDmRD9XzLI3+K/fAbRJw//pYuD4mOd3AaPKWGdPL5rBZ8IylzWZmwccC1wLjLXfQzC/B3+52v7/+dv+81p4f/T2illmBDDUf3wv8FLMa+385evYuuFY13+eB3wJ7AcosGdpOcj6kZNyOhkYBdTDayQei3v9LOAk//UmwDi8Xn994DrgP/5RSi3gUbyGpw5wKDAj5n0OBn4EGgL3A/+KObIZBSzEa6jOAO4VkZ7xhYpIR+BJvEaqOdAAaFHSD6aqf8M7YmwMTBWR90VkkIjUjHvfG0VkbUlTovcWkd2BZniNdJGvgE6JljfFWOaCydyReJ2QIvZ7cPR7EJHDRWQdXuftdOBh/6W9gAJVnVXCe3WK3Y6q/oz/h9LWDc264I1SfqSqX1MO1jkpn09U9W1VLcTrDXaJe/1RVV2gqpuBc/B6/G+r6g5VfQ+YCpzoL7sD6CwiNVR1iarGNoTzVfUZfzvD8XbuJiLSEjgMuEFVt6jqDOBZ4NwEtZ4BvKWqH6nqVuAWf5slUtXPVfUyvAbtSbwGd6GIPBuzzFBVrVfSVMJb1/b/XRczbx1eT9uUzjLnPnMnAN3wRjCK2O/B0e9BVT9R77ROC+ABvFGdovf6NW7x2PeqHbed2Ndt3RCs6+f4EuBWysk6J+WzNObxJqB63DnmBTGPWwNnxh1ZHA40U9WNQH/gUmCJiIwTkQ6JtqOqm/yHtfEajtWquj5m2flAfoJam8fW429zVXl+SL9B+xrviG4b0Lk865Vig/9v3Zh5dfGOjEzpLHOpSTVzNYGL8EY2VsbMt99DalLe91V1ETAeb8So6L3qxi0W+16lvW7rhmPdh4E7VTW+c1Mi65xUjthPliwARsQdXdRS1aEAqjpBVY/DOzL6AXimHO+/GKgvIrFHHa2ARQmWXQK0LHriD9E2KO3NRaSBiFwpIpOB/wG5wNGq2j1mmb9I8U+ZFJsSva+qrvHriT3a7ELxYXOTGstcAqlkzr9IswHwV1X9prS6E20y5rH9HnyVsO/n8duFk7OAPBFpX8J7zYzdjoi0Bar569m64Vj3GOAB8T61VdQRnyQlfKIHsAtiYydKvjgx9oKzPfAapLxE6+A1Dkvxhohzgep4nwZogXdOui/exUM5eBeifeivdx7eEHLstndeNAR8jHe+uzreBUXLirYbWyPeOb4NeEdsVfGGqAvif66YbVyI17t9Be/ceW4l/58OBT4Edgc64DVYvUpYVvyfr6P/s1cHqgWdC8tcRmeuJ97owhL7PQT6exgItPIft/bXezXm9VHAy/7/32F4pww6xfzcvwJH+K+PJObCW1s3FOs2BprGTIp38XiNEvNTmWGM+kQl/KHw5x2Mt3OtBlbgXSTXCu+I6UP/l7YW+ADo6K9zHqU3UC2At/z3/Bm4tJQaB+N9RLU8V+x3BOqn8f+0GvCcH+plwLUxr7XCa0yLGqWi/9vYaV7QubDMZXTm3sf7A74D2Oy/9o79Hpz/Hu7Bu+h3o//vMKBBzPL1gdf9138Bzo7b1tn+/I3AG7E/l60bjnVLyndJk/gLGmOMMcaEgl1zYowxxphQsc6JMcYYY0LFOifGGGOMCRXrnBhjjDEmVKxzYowxxphQidw3abrQsGFDbd16j6DLSNmX3/8SdAkp023r0YLNCb8pNbdua9WCzcWX37xigqqW9E2nkWGZC5ZuXrFSVRslei0+d5mSOfBy1yrCuZuRobnL5LauvKxzkkDr1nvw6RdTgy4jZbsfeGXQJaRs649jSnxNC7dQrePAYvO2THu4YbprcsEyF6wtMx6fX9Jr8bnLlMwBtGq9Bx9PmhJ0GSlrePAfgi6hQkrKXSa3deVlnRMTIQK5FlnjmuXOuGaZy+6f3kSLCOTkBl2FyTaWO+OaZc46JyZKBHKrBF2EyTqWO+OaZc46JyY6xHZYEwDLnXHNMmedExMhAuRm91CnCYDlzrhmmbPOiYkSu0jMBMFyZ1yzzGX3T2+ixYY6TRAsd8Y1y5x1TkyU2BXsJgiWO+OaZc46JyY67GjCBMFyZ1yzzFnnxERMlp+HNQGx3BnXsjxz2f3Tm2ixGxOZIFjujGuWOeucmOjw9lf7Im3jluXOuGaZs86JiRQhN8s/+2+CYLkzrlnmrHNiokNAciToKky2sdwZ1yxzZPe4kWPvThjPfp32plOHPXng/qFBl5PQU7cNZP7E+5j6yl92ee3qQT3Z/OVjNKhXC4B6dWow+u8XM3n0TXw84jo6tmuW1trEP5qInUzZLHcVE587U7qFCxbQ+/ieHNClE926dubxfz4SdEkJJZO5a849hs9H3cjno25k6it/YcPUR9m9bs201WZtnXVOnCksLOSPV13BG2Pf4cuvv+OVUS/z/XffBV3WLkaM/Zy+Vzy+y/wWTepxTPd9+GXJ6p3zrr/wBL76cSEH9b+PC28ZwYN/PiO9xfnnYWMnUzrLXSWIy50pXV5eHvf97UGmfTWT9z+exDNPPcH330c7cw+9MJHuA4bSfcBQbv3nm3w87SfW/LopfcVZW2edE1emTJ5Mu3Z70qZtW6pWrcqZ/Qfw1tg3gi5rF59O/5nV63bd6e6/7nRufuR1VHXnvA5tm/LhlFkAzJq3jNbN69O4fp201SZI1u+wybLcVVx87kzpmjZrRtff7Q9AnTp12LvDPixZtCjgqnaVTOZi9evVjTHjp6W1NmvrrHPizOLFi2jRouXO5/n5LVgUwh02kT499mXx8rV8M6t4vd/MWkTfnl0A6NapNa2a1Se/Sb30FSKQk5NTbDKls9xVgrjcmfKbP28eX331Jd0OOjjoUsqlpMwVqVG9Cscdug+vT5yR3kKsrcuOzomI9BKRH0VktojcGHQ9UVKjehWuv+AE7nxy3C6vPfh/77FbnZp8PupGLhtwFF/9uJDCwh1pqyVKRxOWuYoJc+7CLEy527BhAwMHnMHfHnyIunXrBllKuZSWuSInHbkvk2bMSe8pHaLV1qVLxn9aR0RygceB44CFwBQReVNVnZ4Ebd48n4ULF+x8vmjRQvLz812WkJK2LRrROr8Bk0ffBEB+43pMeukGjhj0AMtWreeS20fuXPaHcXcwd9Gq9BUjROLCsLBkDix3lcJyl7Tt27czsP8Z9B9wNn1POc315lNSVuYAzjzhAF5J8ykdIDKZS6eM75wABwGzVXUOgIiMAvoCTnfYbgceyOzZPzFv7lya5+fzyuhRPD/iJZclpGTm7MW0Puamnc9/GHcHhw28n1VrN7Jb7Rps2rKN7QWFnH/qoXwyfTbrN25Jaz0R+XhdKDIHlrvKYrkrP1Xl8ksuYu8OHfjDH691uekKKS1zAHVrV+fwA/bk/JuHO6knIplLm2zonOQDC2KeLwScnwDNy8vjoUce4/cnnUBhYSGDz7uAjp06uS6jTMPvO48jDmhPw3q1mT3+Lu566m2Gvz4p4bId2jblmTsHoap8//MSLr3jxbTWJhKZGxOFInNguasMlrvkTPrsU15+cQSdOu/LIQf+DoDb77yHE3qf6LqUUiWTOYCTj+7CxM9/YNOWbWmvLUKZS5ts6JyUi4gMAYYAtGzVKi3b6NX7RHqFbAeNN/im50t9vcNJt+18/MXXc9nvlDvTXFGMDLuls4vMgeWuwix3STn0sMPZsDV91wBVlmQyBzBy7BeMHPtFGiuKkWGZS0XoOyci8k8g8We6AFW9qoy3WAS0jHnewp8X/z7DgGEABxzQrcTtmeAIQo6joc4K5s4yl0Fc5S6Itm5/y10ouWzrwir0nRNgagXXnwK0F5E2eDvqAODsCldl3BPIzXW2w1Ykd5a5TOIud9bWGY/bti6UQt85UdViVx+JSE1VLffnuFS1QESuBCYAucBzqjqzkss0DgiQl+dmqLMiubPMZRZXubO2zhRJJXMi8hzQB1iuqp39efWB0cAewDygn6quEREBHgFOBDYB56nq9MqqvzJE5qSWiBwiIt8BP/jPu4jIE+VZV1XfVtW9VLWdqt6T1kJN2ohATo4Um8peR54TkeUi8m3MvPoi8p6I/OT/u7s/X0TkUf8eEV+LyP6p5s4ylznic1e+dSqUu8HW1mW3VNo64HmgV9y8G4GJqtoemOg/B+gNtPenIcCTlVJ4JYpM5wR4GDgBWAWgql8BRwZZkHFNyM3NKTaVw/NUbId9GMtdlpNkMwcVy93jWOayXPJtnap+BKyOm90XKBqRGw6cEjP/BfV8DtQTkfR+g2aSotQ5QVUXxM0qDKQQEwjxz8PGTmWp6A4LVLHcZbf43JVHBXOXCxTErWuZyyIltHUNRWRqzDSkHG/VRFWX+I+XAk38x4k+dh6quzOG/pqTGAtE5FBARaQKcDXwfcA1GccSDG82FJHYCwmH+Z9GKE0yOyyWOxOXu1QyB+XP3Xq80RTLXBZL0NatVNVuqb6fqqqIRObTWVHqnFyKdwFPPrAY76KvKwKtyDjl3Zhol8G+dO+w9+LlzHKXpRLkrkKZgzJz9wNwJpa5rFVCW5eKZSLSTFWX+Kdtlvvzy/Wx8yBFpnOiqiuBgUHXYQJUeR+vS2aH/U5VLXfZzH3umgA9YkZZTLapvMy9CQwGhvr/vhEz/0r/Kw4OBtaFLW+RueZERNqKyFgRWeFfBf+GiLQNui7jjlBpXyNetMPCrjvsuf6nJ7oD64AalrvsFp+7Cihv7rYAwyxz2SuVtk5EXgYmAXuLyEIRuRCvU3KciPwEHOs/B3gbmAPMBp4BLk/Dj1EhkRk5AV7Cu4r9VP/5AOBlAvrOEhOAFI4m/B22B951AguB2/B20DH+zjsf6Ocv/jbe5/5n4332/3wsd8Z97gDGYJnLXilkTlXPKuGlYxIsq4T8VGGUOic1VXVEzPORIvLnwKoxzglCXpLnYSu6w/o3wrLcZTHXuRORry1z2S2VzGWa0HdO/DvcAbwjIjcCo/C+f6I/3hGHyRJFNyZysy3LnfG4yp1lzhRx2daFVeg7J8A0vB206Dd1ScxrCtzkvCITmDx33zdhuTM7OcqdZc7s5LCtC6XQd05UtU3QNZhwEBHyKnZBYrlZ7kwRV7mzzJkiLtu6sAp95ySWiHQGOgLVi+ap6gvBVWRcEiA3gKFOy112CyJ3lrnsFlRbFyaR6ZyIyG14V793xDv/2hv4BLAdNktIClewV3yblrts5zp3ljkTRFsXNlEaNzoD70r3pap6PtAF2C3YkoxreTk5xSYHLHfGMmecC6CtC5XIjJwAm1V1h4gUiEhdvLsrtixrJZM5RCSIoU7LXZYLIHeWuSwXUFsXKlHqnEwVkXp4d7ObBmzAuxueyRJCIFewW+6yXAC5s8xluYDaulCJTOdEVYtur/uUiIwH6qrq10HWZNwSwfnwpuXOuM6dZc4E0daFTeg7JyKyf2mvqep0l/WYYDm8CZvlzuzk6CZsljmzk92ELfz+XsprCvR0VYgJloi4HOq03BnAae4scwZw3taFUug7J6p6dNA1RE5u6H+tJZOSd0jB3VCn5S5JUc5cGVzlLojM2f00wsllWxdWmduimIyUl937qwmI5c64lu2Zs86JiQwRu4LduGe5M65Z5qxzYiJEgCo2BG0cs9wZ1yxzEbpDrHjOEZFb/eetROSgoOsy7hTdmCh2crBNy12Wi8+dg+1Z5rJcEG1d2ESmcwI8ARwCnOU/Xw88Hlw5xjUBquRKsckBy12Wi8+dA5a5LBdQWxcqUTqtc7Cq7i8iXwKo6hoRqRp0UcYdkUCGOi13WS6A3FnmslxAbV2oRKlzsl1EcvE+74+INAJ2BFuScS2A4U3LnXGdO8ucycpTObGidFrnUeA1oLGI3IP3FeL3BluScUkkkKFOy12Wi8+dA5a5LBdQWxcqkRk5UdUXRWQa3leJC3CKqn4fcFnGIUGcD3Va7ozr3FnmTBBtXdhEpnMiIq2ATcDY2Hmq+ktwVRmXRCDX8Vif5c64zp1lzgTR1oVNZDonwDi8c7ACVAfaAD8CnYIsyrjjXcHufI+13GW5AHJnmctyAbV1oRKZzomq7hv73P8Gz8tLWNxkIO9rxJMf6hSReXgfxywEClS1m4jUB0YDewDzgH6quiZ+XcudSSV3ljlTEUG0dWET2a6Z//XhBwddh3FHgFyRYlMSjlbVrqrazX9+IzBRVdsDE/3nZbLcZZ/43CXBMmdSEoa2LmiRGTkRkWtjnuYA+wOLAyrHBKJS75TYF+jhPx4OfADcsMsWLXem8nJnmTPl5L6tC5sojZzUiZmq4Z2X7RtoRcYpAfJEik1AQxGZGjMNSbCqAu+KyLSY15uo6hL/8VKgSQmbtdxlufjcYZkzaRZQWxcqkRg58W9IVEdVrwu6lop4d8J4rrv2agoLCznvgov48/XhG1176taz6H14R1as2UC3/n8r9trVA3sw9JpTaHHMzaxat5FrBh1N/17eyGFeXg4d9mhCy+P+yppfN6WlNhESDW+ujBm+LMnhqrpIRBoD74nID7EvqqqKiO66PcudKxHLnWWuDJHI3G0D6X1kZ1asXk+3M4vfRubqQT0Zeu1ptDj6Blat3cg15x5D/xMPBCAvN4cObZrSsueNGdPWhVHoR05EJE9VC4HDgq6lIgoLC/njVVfwxth3+PLr73hl1Mt8/913QZe1ixFjv6DvH57eZX6LJvU4pnsHflmyeue8h0a8T/eBD9B94APc+thbfDx9dtp21iKpfBmWqi7y/12Od3Org4BlItIMwP93eew6lju3opS78rDMRSFzn9P3il2/ssjL3D7FM/fCRLoPGEr3AUO59Z9v8vG0nzKmrQur0HdOgMn+vzNE5E0RGSQipxVNgVaWhCmTJ9Ou3Z60aduWqlWrcmb/Abw19o2gy9rFp1/OYXWCne7+a0/h5kffREvoc/c7YX/GTJie1toEIU9yik1lriNSS0TqFD0Gjge+Bd4EBvuLDQbifxmWO4eilLsyl7fMRSNz039m9boEmbvudG5+5HW0hND169WNMeOnpbU2x21dKEXitI6vOrAK6Mlv9wBQ4NUgiyqvxYsX0aJFy53P8/NbMHnyFwFWVH59jurM4uXr+OanxNfk1ahWheMO6cA19/8nrXUUXcGepCbAa+Ktlwe8pKrjRWQKMEZELgTmA/1KWN9yF5AI584yF9XM9diXxcvX8s2sRQlfr1G9Cscdug/XDB2T1joCautCJQqdk8b+1evf8tuOWqTMc2ci8hzQB1iuqp3TU2LmqlGtCteffxx9rniyxGVOOrIzk76am/ZhTiT5L8NS1TlAlwTzV+HdHrwklrsARTl3lrloqlG9CtdfcAJ9Ln+sxGVOOnJfJs2YE7rMQYVyF0pROK2TC9T2pzoxj4umsjwP9EpXceXVvHk+Cxcu2Pl80aKF5OfnB1hR+bRt0ZDWzesz+eXr+eHNW8lvvBuTXryOJg3q7FzmzON/xytpHloH/wr2HCk2pZHlLkBhzl0aWeYC1LZFI1rnN2Dy6Jv4Ydwd5Deux6SXbiieuRMO4JU0n9IB521dKEVh5GSJqt6Z6sqq+pGI7FGJ9aSk24EHMnv2T8ybO5fm+fm8MnoUz494KeiyyjTz5yW0Pv6Wnc9/ePNWDhv0d1at2whA3VrVOXz/dpx/y8i015LiUGeqLHcBytLcWeYCNHP2Ylofc9PO5z+Mu4PDBt7PqrV+5mpX5/AD9uT8m4envRbHbV0oRWHkJCN+Q3l5eTz0yGP8/qQT6LrvPpx+Zj86dgrfV2UMv+dcPvi/q9mrdWNmj7udwX1LvzHlyUfvx8QvfmTTlm3pLy7ujolp3nktdw5FKXfp3FI639yVyGTuvvP4YPif2Kt1E2aPv4vBpxxS6vInH92FiZ//kIltXShJSVckh4WI1FfV1WUvWep77AG8Vdp5WP+GNUMAWrZqdcCsn+dXZJOB2r37H4MuIWVbv3+ZHRuXJdwT99n3d/p/r79fbN4he+4+rRyf/U+ai9xZ5sJjy7RHSsxRfO6inDl/mczJ3YFXBl1ChWyZ8XjCLLls68Iq9CMnFd1Zk9jOMFXtpqrdGjVs5GKTJgU5IsWmdHGRO8tcdGRK5vztWO4iwFVbF1ZRuObEGMAb887GndQEy3JnXLPMRWDkpKJE5GVgErC3iCz0P+ttokggJ6f4FFaWuwwiljnjWITaunTJ+JETVT0r6BpMZYnO8KblLpNEI3eWuUwSjcylU8Z3TkzmsKFOEwTLnXHNMmedExMlkp3DmyZgljvjmmXOOicmOuxowgTBcmdcs8xZ58RETLbvsCYYljvjWrZnzjonJlKyfajTBMNyZ1zL9sxZ58REhogdTRj3LHfGNcucdU5MpNjH60wQLHfGNcucdU5MZAg21Gncs9wZ1yxz1jkxEZPtRxMmGJY741q2Z846JyY67DysCYLlzrhmmbPOiYkOAXKzfKjTuGe5M65Z5qxzYiJGsvxowgTDcmdcy/bMWefERIYI5Gb5Dmvcs9wZ1yxz1jkxEZPtV7CbYFjujGvZnjnrnJjIEPvsvwmA5c64ZpmDLO+bmajJESk2lUVEeonIjyIyW0RudFCiyUDJZA4sd6bikm3rILNyZ50TExnif4147FT68pILPA70BjoCZ4lIx/RXajJJfO7KXt5yZyom2bbOWyezcmedExMpSR5NHATMVtU5qroNGAX0TXuRJuMkeQRruTMVlsLISUblzjonJlKS3GHzgQUxzxf684xJSpJ/JCx3psJS6JxkVO7sgtgEpk+ftrJGFZmfxk00BFam8f3TKd21ty7phS+nT5tQs6o0jJtdXUSmxjwfpqrD0lNa+ljmyhSm3GVE5sByVw6B5C6T27ryss5JAqraKJ3vLyJTVbVbOreRLkHWrqq9klxlEdAy5nkLf17oWOZKZ7lLD8td6YKqP4XMQYRyVx52WsdksilAexFpIyJVgQHAmwHXZDKf5c4EIaNyZ52TMojI7SIy0sF2zhORT1Jct9QaRWSeiBybenXRpKoFwJXABOB7YIyqzgy2qrJZ5qLNclfmdix3aRDV3JUk6zsnIrIhZtohIptjng9M02ajfJ4wqdpFpJqIPCciv4rIUhG5tpzrTRQRFZEKnXpU1bdVdS9Vbaeq91TkvSqLZS4l5a4/mcz5fygL434nPSparOVuJ8tdycu3FZG3RGS9iKwUkfsrWmwYc5eqrO+cqGrtogn4Bfh9zLwXk3mv8v4hDdNFTCLSSJL4hqkUar8daI934dfRwPUiUur5VL+hrJLkdiLDMpdc5iDp+m8nucxNiv2dqOoHydQWFZa78OTOP+3yHvA/oCne9SFpH7WKkqzvnJRTVRF5we/hzhSRnRdI+cOIN4jI18BGEckTke4i8pmIrBWRr2KPxPwjtTn+e82NP2IRkQdFZI3/Wu+Y+c1F5E0RWS3e3f8uLqlYERkkIvNFZJWI3FzGz3YBMFdE7hCRNkn+v5THYOAuVV2jqt8DzwDnlbSwiOwG3AZcn4ZaosQyl7qkMmeKsdylLpncnQcsVtV/qOpGVd2iql+noabIss5J+ZyMd0ObengXGD0W9/pZwEn+602AccDdQH3gOuA/fq+9L/AvIBe4BzgUmBHzPgcDP+J9fO1+4F8xPf1ReJ9bbw6cAdwrIj3jCxXvjoBPAoP8ZRvg9coTUtW/4V041RiYKiLv+zt8zbj3nSLeUHCh3xAVmxK9t4jsDjQDvoqZ/RXQqaR6gHv9+peWskw2qKzM1cK7ayTAMuAlopO5G0Vka0m5S/TeKWbud+INq88SkVukgqcSI87aOje56w7ME5F3/Ox9ICL7llR7VlJVm/wJmAccGzfvduC/Mc87Apvj1rkg5vkNwIi495iA11OeA/wK9Ae+BjrGLHMe3t39ip7XBBRvyK8lUAjUiXn9PuD5mBpH+o9vBUbFLFcL2Bb/c5Xw81cD+gFvA6uBZ2NeOxLYH/g2if/Plv7PUD1m3nHAvBKW74bXgOUBe/jr5gWdiwhnbjBQx8/OZUBdvAazYxQyl0ruUshcW6AN3oHavsB3wE1B5yLiuTuPCLd1jnL3LrAd71bzVYE/+/9nVYPORlgmGzkpn9ij+E14N8OJPbqKvStfa+DMuJ724Xi3Fv4JOBO4CO/c5Ksi0iHRdlR1k/+wNt5RwWpVXR+z7HwS3/2veWw9qroRWFWeH1JVt+I1JDPwdvLOMa99hLcTJ2OD/2/dmHl1gfXxC4pIDvAEcLV6V51nu8rIXDO83+GXwGl41xlUw8vfLtsJW+b815PNXbkz57//HFWdq6o7VPUb4E68o/VslfVtnf96WnMHbAY+UdV31LvV/IN4Iz/7JLHNjGadk8qhMY8X4B1N1IuZauFd+LRAVSeo6nHAFXgBfaYc778YqC8idWLmtSLxDXaWEHMjHn/IskFpby4iDUTkShGZ7NeZCxytqt1jlvkLMBPYR4pf9b9BRDYkel9VXePX0yVmdhf/feLVxRs5GS0iS/E+sw+wUESOKK3+LFVm5lR1KF6jPsPPXDO8o7MB5Xj/UGTOz1bC3CV63yQzl/AtgOz+rvrSZUVb5yB3X1P8/9LEsc5J5RsJ/F5EThCRXBGpLt5FYvXxjkL6+tcBFOAN6+0o6w1VdQHwGXCf/377AReS+OrufwN9RORw8a4Iv5NSfs8iciHecO1RwB1AS1W9Qb0LumJruBfv/On3WvyTDUVX/5fkBeCvIrK7f+R0MfB8guXW4R0JdfWnE/35BwBflPL+poTMiUgLYDeglZ+5rf5UZqMYlsz52UqYu1LKL2/mEJHeItLEf9wBuAV4o5T3Nr/J2LYu3bnzf57uInKseN8m/Ee82+R/X8LyWcc6J5XM37n6An8BVuAdXfwZWI53FHst3tHB03gXg11Wzrc+C+86jMXAa8BtqvrfBNufiXek8hJeT34N3sVlJZkEtFbVM1V1nKoWlrOe8roN+BlvaPZD4AFVHQ8gIq38o5FW6llaNOH93wEs84c9TQlKyVwO3kWw3fBysxrvaG5MOd86ozPnL3sM8LWIbMS7/uBVvIuyTRmsrdtFuXOnqj8C5wBP+XX3BU62tu43omojSy74521n4TWGi/BOW5ytEbqDn4jsAbylqp3LWtaEg+XOuJYJmQPLXdBs5MQRjfithUXkZbwjj71FZKE/RGpCznJnXIt65sByFwY2cmKMMcaYULGRE2OMMcaEinVOjDHGGBMq1jkxxhhjTKhk83dIlKhOvfrasHmJX9EQeivXR/fTaFtWL2X7xrUJb4KVW7e1asHmYvN084oJqlrqtxxHgeTVUKlWt+wFQ6rL3i3LXijEZnw5baWqNkr0WnzuMiVz4Oeuap2yFwyp3+3TquyFQmz69MS5y+S2rrysc5JAw+YtuOOFcUGXkbJnPpwfdAkpm/HwRSW+poVbqNax2BebsmXaww3TXZMLUq3uLj9blHzw0QNBl1Ah9WrmlbjTxOcuUzIHIFXrUG3vfkGXkbJPv4j/XsJoqVFFEuYuk9u68rLOiYkQgZzcoIswWcdyZ1yzzFnnxESHCORaZI1jljvjmmXOOicmSgRyqwRdhMk6ljvjmmXOOicmOsR2WBMAy51xzTJnnRMTIQLkZvd5WBMAy51xzTJnnRMTJXYe1gTBcmdcs8xl909vosWGOk0QLHfGNcucdU5MlNjH60wQLHfGNcucdU5MdNjRhAmC5c64ZpmzzomJmCw/D2sCYrkzrmV55rL7pzfRIjbUaQJguTOuWeasc2Kiw9tf7Yu0jVuWO+OaZc46JyZShNws/+y/CYLlzrhmmbPOiYkOAcmRoKsw2cZyZ1yzzFnnJN3+dPKhVK9Zi5ycXHLycrnjhXG8NuwffPD6y9St1wCAM664ni6H9Qy40sTO+F1zfr9fEwQY+80yXpm+eOdr/Q/I58oebejzxOes21yQ9lrEjiZK9NQt/el9eEdWrNlAtwEPAHDrpb3oc2RndqiyYvUGhtzxMktW/soR+7fjlb9fwLzFqwF44/1vuO/Zd4Msv5grLrmICePH0ahRYyZN/arYa/985B/cctP1/PzLUho0dPMN8pa7xJ66bSC9j+zMitXr6XbmvcVeu3pQT4Zeexotjr6BVWs3AnDEAe154M+nUyUvl1VrN3D8RY8EUXaZtmzZwrFHH8m2rVspKCzg1NPO4Jbb7nBag2XOOidO3PjUaOrUq19s3glnXcSJgy4JqKLyadOgJr/frwlDXvyKgsIdPHh6Zz6bs5pFa7fQuE5VDtqjHkt/3eKuIDsPW6IRb03hqTGf8OwdZ++c99CI97nzqfEAXN7/CG666HiuGvpvAD79cg6nX/uvQGoty9mDzuXiSy/nsovPLzZ/4cIFvD/xPVq0bOW2IMtdQiPGfs5Toz/k2bvOLTa/RZN6HNN9H35ZsnrnvN1q1+CRv/Sj7xVPsGDpGhrtXtt1ueVWrVo1xr/3P2rXrs327dvpedThHH9Cbw7u3t1dEZY5svunN6Vq3aAG3y1Zz9aCHRQqzFi4jqPae6M9f+jRlic+moequ3oEIScnp9hkPJ9+OYfVv24qNm/9xq07H9esURV1+cuqgMMOP5Ld69ffZf5frv8Td9w9FBG3w93xuTOeT6f/zOp1m3aZf/91p3PzI68Xy1v/3t14Y+JXLFi6BoAVazY4qzNZIkLt2l7nafv27RRs3x545rIxd9n3E7smwgNXnsOtg07k/Vdf3Dl74ivDufms43n2zuvY+Ova4OorxdyVm+iSvxt1q+dRLS+H7m12p3Gdahzerj4rNmzj5xUb3RbkH03ETqZ0t1/Wm5/euoUBvfbnrqfH75x/8L578MWL1/H6IxezT9smAVZYPuPGvkmz5vnsu18X9xuPy50pWZ8e+7J4+Vq+mbWo2Pz2rRtTr25NJjxzNZ++eD1n9zkooArLp7CwkIMP6Eqr5o3peexxHHTwwW4LsLYuOzonItJLRH4UkdkicqPLbd/8zH+4c+TbXPfIC0z89wv8MP0Lep4+iAde+5i7XhxPvYaNefnhu12WVG7zV2/mxSkL+ccZnXnw9E7MXr6RKrk5DDq4Jf/6dL7zegSJzA4bZOZi3f7kO7Tvcxejxk/n0n6HAzDjx4XsffJdHDzwQZ4c/QljHrggqPLKZdOmTfzjgfv4yy23B7L9+NyFWZC5q1G9CtdfcAJ3Pjlul9fycnPYf5+WnPqHJzn5ise56eJe7NmqscvykpKbm8sX02Ywe95Cpk6ZzMxvv3W6/Si1demS8T+xiOQCjwO9gY7AWSLS0dX26zduCkDd+g05oMcJzJk5g90aNCInN5ecnByOOuUs5syc4aqcpI37dhkXjZzBH0Z/w/qtBcxduYlmu1Xj/879HWMu6kajOtX41zldqV/Twa2WhUgMdQaduURGvzONU3ruB3inezZu3gbAhM++p0peLg12qxVkeaWaO+dn5s+fx+EH78++HdqxeNFCjjr0QJYtXeqmgLjchVXQuWvbohGt8xswefRN/DDuDvIb12PSSzfQpEEdFi1fy3uTvmfTlm2sWruRT6bPZr+98l2VlrJ69epxVI+jeffd8WUvXJki0talUzb8xAcBs1V1jqpuA0YBfV1seOvmTWzeuGHn428//5gW7fZm7cplO5eZ9sEEWrTb20U5KalXw+t0NK5TjSPbN2D8d8s4+cnJ9Ht2Kv2encqK9Vu5cOQMVm/a7qSeiBxNBJa5WO1a/vZplj5HdWbWvOUANGlQZ+f8bh1bkZMjrFrn+BRdEjp13pfZ85fwzQ8/880PP9M8vwUffjaFJk2bOqshApmDgHM3c/ZiWh9zEx1Ouo0OJ93GouVrOeTsv7Fs1XrGfvA1h3ZtR25uDjWqV+HAznvww1xHncskrVixgrVr1wKwefNmJv73Pfbeu4PzOiLS1qVNNnxaJx9YEPN8IeDkBOK6VSt49PohABQWFHBIr1PY79AePH3r1fwy6zsQoWGzFpz/l/tclJOSu0/uwG41qlBQqDw08Wc2bC0MrBaRyHy8znnmht99DkccsCcN69Vi9lu3ctewCfQ6bB/at27Ejh3KL0vXcNV93id1Tu3ZhYvPOJSCgh1s2bqdc28ekc7Sknbh4IF88tGHrFq1ko57tubGv97GuecFd+rJcpfY8PvO44gD2tOwXm1mj7+Lu556m+GvT0q47I9zl/HeZ98xZcxN7NihPP/aZ3z385J0lVYhS5cs4eILBlNYWMgO3cHpZ/TjxJP6OK0hQplLm2zonJSLiAwBhgA0aFo5w42NW7Tm7pcm7DL/kjvD+fn+RK4c/U2pr/d7dqqjSjyZdGOi2MxRtU7pC5dh8F9H7jJv+JtfJFz2qVc+4alXPqnQ9tLpX8NfLPX1b3742VElv0k2dyLyHNAHWK6qnf159YHRwB7APKCfqq4R76MgjwAnApuA81R1eqUVv2ttv+WuSuof6R180/Olvt7hpNuKPX/ohYk89MLElLfnyr777cfnU78MuoyMautSkQ1jRYuAljHPW/jzilHVYaraTVW71dl9148xmuB5RxM5xaaQSjpzklfDWXEmOfG5K6fngV5x824EJqpqe2Ci/xy8a0Ta+9MQ4MkUS7XcZYgItXVpE/qRExH5J1DiDRpU9aoy3mIK0F5E2uDtqAOAs0tfxYSSQG6usyPYesB7wIpE71tG7ixzmSSF3KnqRyKyR9zsvkAP//Fw4APgBn/+C8CjeG3dXiLyLN4oSux7WluXLVLIXKYJfecEqNB5A1UtEJErgQlALvCcqs6slMqMU0JKO+zzwGN4jX+RoiPYof7HLW/E+yMRewR7BzAQuDPZDVrmMkuKuUukiaoWXWixFCi6wUzRtSJFbd0x/vN5yby55S5zpJK5MJ9KTEXoOyeqOjz2uYjUVNVdb0tY+nu8DbxdqYUZ50RIengzlSNY9W5teauI9AfeVdUlyebOMpc5EuSuoYjEHjQNU9VhybynqqqIaNy84d725FxgHPCdtXXZKZW2jtQPxA7GO5Xo+E5zpYvMiSwROUREvgN+8J93EZEnAi7LOJXwPGxDEZkaMw0pxxuVdQRbZCHQy3KX7Xa55mRl0TUb/lTejskyEWkG4P+73J8ff63InsBLWOayWPLXnKjqR8DquNl98Q7A8P89JWb+C+r5HKhXlM2wiEznBHgYOAFYBaCqXwFHBlmQcUsEcnKk2ETqfygA7wiWUq5pAv6E5S6rxeeuAt4EBvuPBwNvxMw/VzzdgQZ4p3Ysc1mqhLYuFckciIXqrnihP60TS1UXxH0BU3A33TCBqKRz/8tEpJl/uqa0I9gWwEbLnUnh/P/LeKcOG4rIQuA2YCgwRkQuBOYD/fzF38Y79z8b7/z/PMucSZC5Cp1OTHQqMcyi1DlZICKHAioiVYCrge8Drsk4VPTxukpQdAQ7lF2PYK8UkVF451/XAb9Y7rJbKrlT1bNKeOmYBMsqcEXM9v5tmctuJWRupap2S/KtkjkQ2+Vj50GK0mmdS/F24HxgMdCVmB3aZIdkhzr9I9hJwN4istA/ah0KHCciPwHH+s/BO4Kdg3cE+wxwOZY7Q6Wd1ikvy5yprNM65T2VuC7m9E8oRGbkRFVX4n2002SpFD+tk/IRbAzLXRZL8ZMTKbO2zqSSuQqeSjy/ciqvPJHpnIhIW7zPZXfHu4BxEnCNqs4JtDDjTgA3JrLcGde5s8yZFG/8VxkHYqERpdM6LwFjgGZAc+AV4OVAKzJOCVJZQ53JsNxlufjcOWCZy3IBtXWhEqXOSU1VHaGqBf40EqgedFHGHRHIy80pNjlgucty8blzwDKX5QJq60Il9Kd1/NvvArzj3+FuFN5QZ3/sTohZJ8/R8LrlzsRykTvLnInlqq0Lq9B3ToBpeDto0W/qkpjXFLjJeUUmEALkirMd1nJnAKe5s8wZwHlbF0qh75yoapugazDhICLOhjctd6aIq9xZ5kwRl21dWIW+cxJLRDoDHYk5/6qqL5S8hskkEtDXiFvuslsQubPMZbeg2rowiUznRERuw/sMd0e886+9gU8o/g2MJsO5Huq03BlwmzvLnAE7rROlcaMz8D6vvVRVzwe6ALsFW5JxqWio0/EV7Ja7LBefOwcsc1kuoLYuVCIzcgJsVtUdIlIgInXxviOgZVkrmcwhBHIFu+UuywWQO8tclguorQuVKHVOpopIPbzvPJkGbMC7c6LJIjnuhzotd8Z17ixzJoi2LlQi0zlR1cv9h0+JyHigrqp+HWRNxq2iGxO5ZLkzrnNnmTNBtHVhE/rOiYjsX9prqjrdZT0mON55WGc3YbPcGcBd7ixzpojLti6sQt85Af5eymsK9KzsDdavUZV+XaN7iveSi/8WdAkp27pqdamvO7yC3WnuftehJZ9O+kdlvqVTux94ZdAlpJWj3Dlv6zrv1ZI333uwst/WmfoDngu6hLTJ9k/rhL5zoqpHB12DCQcBqjg6mrDcmSKucmeZM0VctnVhFfrOiTFFvPOw2b3DGvcsd8Y1y5x1TkyECJCX5UOdxj3LnXHNMmedExMhdpGYCYLlzrhmmYvQHWLFc46I3Oo/byUiBwVdl3Gn6Dxs7JT2bVrusl587tK+Pctc1guirQubyHROgCeAQ4Cz/OfrgceDK8c4J94V7LGTA5a7bBeXOwcsc9kumLYuVKJ0WudgVd1fRL4EUNU1IlI16KKMOwFdwW65y3IB5M4yl+Xs0zrR6pxsF5FcvM/7IyKNgB3BlmRcEglkh7XcZbkAcmeZy3IBtXWhEqXTOo8CrwGNReQevK8QvzfYkoxLAuTmFJ8csNxlufjcOWCZy3IBtXWhEpmRE1V9UUSm4X2VuACnqOr3AZdlHBIRquS4PZqw3BnXubPMmSDaurCJTOdERFoBm4CxsfNU9ZfgqjIueedh3R5CWO6M69xZ5kwQbV3YRKZzAozDOwcrQHWgDfAj0CnIoow7AqRyGlZE5uF94qEQKFDVbiJSHxgN7AHMA/qp6poEq1vuslyquasAy1yWCyBzoROZzomq7hv73P8Gz8tLWNxkIBHIy0n5aOJoVV0Z8/xGYKKqDhWRG/3nN8SvZLkzFcxd0ixzJtXMVfBALFQiO27kf334wUHXYVwScnOKTxXQFxjuPx4OnFKelSx32Sj5zInIPBH5RkRmiMhUf159EXlPRH7y/929PO9lmctGFWrrjlbVrqrazX9edCDWHpjoPw+9yIyciMi1MU9zgP2BxQGVYwLgDXWm1CFR4F0RUeBpVR0GNFHVJf7rS4EmCbdpuct6FchdSqN1ljlTgcwl0hfo4T8eDnxAgtyFTWQ6J0CdmMcFeOdl/xNQLSkrLCzksIO70Tw/n1ffeCvocnbx1G0D6X1kZ1asXk+3M4t/evHqQT0Zeu1ptDj6Blat3UifHvty62V92KFKQeEOrn/g33w2Y07aahNJ+GVYDYuOTH3D/M5HrMNVdZGINAbeE5EfYl9UVfU7LolELneXXHQB77z9Fo0aN2bajG8BWL16NYPO7s/8+fNo3XoPRr48ht13L9eBuxOJcnfr5SfR56j92KHKitXrGXLbSJasWMc15x5D/xMPBCAvN4cObZrSsueNrPl1U1pqKyF3qSjvH4nIZW7xogX86YqLWLliOSLCWYMu4PxLrmTtmtVcefEgFv0yn/xWrXn82ZHsVi8cuXvq8sPpdUBLVqzbwoHXvgbAPYMO5MRuLdlWsIO5S9dzyeMfs27TNvof0ZZrTv7tbFvn1vU59Po3+Hre6rTUVoG2LuUDsbCJxGkd/4ZEdVT1Dn+6R1VfVNUtQdeWrMcefYS999kn6DJKNGLs5/S9Ytc7ZbdoUo9juu/DL0t+2xnf/+JHDup/H90HDOXS20fyxK1np72+BEOdK1W1W8wUv7Oiqov8f5fj3T/iIGCZiDQD8P9dHr9eVHM3aPB5vPHW+GLzHrx/KD16HsO33/9Ej57H8OD9QwOqLrFEuXto+MSd+Xrn42+5aUhvb/4LE+k+YCjdBwzl1n++ycfTfkpbx6RIXOYaisjUmGlIglWK/khMi3m9zD8SUc1cXm4eN98xlPc+/ZJXx3/IC889zU8/fs+Tjz7IYUf04P3J33LYET148tEHgy51pxHv/8Qpd79bbN7/vl5Et2te4+A/vc5PS9Zx3Wn7ATD64zl0//MbdP/zG1z4z4+Yt3x92jomRVJp6/AOxPYHegNXiMiRsS+qquLf3C/sQt85EZE8VS0EDgu6lopauHAh498Zx/kXXBR0KSX6dPrPrF63a0N//3Wnc/Mjr+Nl27Nx87adj2vVqIamOfJFQ53JfN+EiNQSkTpFj4HjgW+BN4HB/mKDgTfi1ots7g4/4kjq169fbN5bY9/gnEHej3vOoMGMffP1ACorWaLcrd/429/jmjWqFctekX69ujFm/LS01hafO9L0RyLKmWvctBmdu/wOgNq167DnXh1YumQx773zFqf3PweA0/ufw7tvjy3tbZz69PtlrN6wtdi8iV8tpnCH92uZMmsF+Q1q7bJev8Pb8u9P56a1tlTaOkj9QCyMonBaZzLeOdcZIvIm8AqwsehFVX01qMKS9ec//ZF77rufDRvWB11KUvr02JfFy9fyzaxFu7x28tH7cecfTqZR/TqcdtVTaa1DEPIk6f50E+A18XbuPOAlVR0vIlOAMSJyITAf6Be3XsbkDmD5smU0a9YMgKZNm7J82bKAKyqf26/4PQP7HMS6DZvpNeTRYq/VqF6F4w7dh2uGjklrDankLvaPhIgU+yOhqktK+COREZlb+Mt8vvtmBl0POJCVK5bTuKmXu0ZNmrJyRST+LgJwbs/2CTshpx/ahn5/+29at51K5vyDrxxVXR9zIHYnvx2IDSXBgVhYhX7kJEZ1YBXQE+gD/N7/t1Qi8pyILBeRb9NcX6neHvcWjRs1Zv8DDgiyjKTVqF6F6y84gTufHJfw9Tff/5qup91Nv2uHcevlJ6W3GEk41FkqVZ2jql38qZOq3uPPX6Wqx6hqe1U9VlVLGqONdO4SEREkIt9yevvjY2nf+xZGvTOVS/sXG3zgpCP3ZdKMOWk/pROfuzIXT3G0LkZkM7dxwwYuO/8sbrn7AerUqVvstSjl7vrTulBQqIz6+Odi8w9s34hNWwv4bsHa9BaQQluHdyD2iYh8hdfRHaeq4/E6JceJyE/Asf7z0IvCyElj/+r1b/ntxkRFynMi4XngMeCFyi+t/CZ99ilvvfUm48e/zdYtW/j11185/9xz+L8XRgZZVpnatmhE6/wGTB59EwD5jesx6aUbOGLQAyxb9dsI0KfTf6ZNfkMa1KvFqrUbS3q7CqnkK9jLkhG5K9K4SROWLFlCs2bNWLJkCY0aNw66pKSMfnsKr/3zMu5+6u2d88484QBeSfMpHUgpd6mO1kU6c9u3b+ey88+i7xn96dXnFAAaNmrM8qVLaNy0GcuXLqFBw0ZBlJaUc3rsSe8DWnLiHe/s8toZh7XhlU/Td9F/kVTaOlWdA3RJMH8V3lchREoURk5ygdr+VCfmcdFUKlX9CEjvlUvlcNc99/HzvIX8OHseL7w4ih5H9wx9xwRg5uzFtD7mJjqcdBsdTrqNRcvXcsjZf2PZqvW0bdlw53JdO7SgWtW8tHVMwNth83Kk2JRGGZG7Iif1OZmRI7zbuowcMZw+v+8bcEVla9fqtz9kfXrsx6x5v52Kqlu7OocfsCdjP/g67XXE564sFRiti2zmVJUb/ngpe+61NxdddvXO+cf2Oon/jPbauf+MHslxvcscAArUcV3zuabvvpz5t/+yeVthsddE4PRD2vDKJ+m93gSct3WhFIWRkyWqemfQRWSL4fedxxEHtKdhvdrMHn8Xdz31NsNfn5Rw2VOP6crZfQ5me0EhW7ZuZ9ANz6W3uCQuDKsEkc3dueecxccffsDKlStpt0cLbrn1Dq67/kbOOasfw//vX7Rq1ZqRL6f3Oo1kJcpdr8M70b51Y3bsUH5Zspqr7hm1c/mTj+7CxM9/YNOWbaW8ayVxl7vIZm7qF5/x2piX2LtjZ07s4d0v7s8338FlV13HlRedw5gXh5PfshWPPRueA7Ln/9iDIzs1pUGd6vz0dH/uHj2d607tQrUqObx1ywkATP5pBVcN+wyAwzs2ZeGqjcxb7uCaQbdtXShFoXPi5Dfkf9xvCEDLVq3Suq0jj+rBkUf1SOs2UjX4pudLfb3DSbftfPz35//L359P74VhsRyf1kn7htKVuRdGvpxw/jvvTqy0bVS2RLkrqVMMMHLsF4wc+0UaK/qNw9w5b+uat2hZKe95YPfDmLtic8LXXnx119MjYXDewx/sMm/4/34qcfmPZy6lx1/c3JvKcVsXSlE4rePkXJmqDiv6aGCjCJwXzVY5OVJsSqO0584yFx2ZkjkonrsGDSx3YeWwrQul0I+clPIpCpNlBMhxdDRhuTNFXOXOMmeKuGzrwioKIycVIiIvA5OAvUVkoX+lvIkigZy4KawsdxnEMmdci1Bbly6hHzmpKFU9K+gaTGWJzvCm5S6TRCN3lrlMEo3MpVPGd05M5rChThMEy51xzTJnnRMTMVl+MGECYrkzrmV75qxzYiJDhKwf6jTuWe6Ma5Y565yYiMn2oU4TDMudcS3bM2edExMpWX4wYQJiuTOuZXvmrHNiIsOGOk0QLHfGNcucdU5MpEjWD3WaIFjujGuWOeucmMjwPl4XdBUm21jujGuWOeucmIjJ9qFOEwzLnXEt2zNnnRMTHWJXsJsAWO6Ma5Y565yY6LChThMEy51xzTJnnRMTMdk+1GmCYbkzrmV75qxzYiJDBHKzfKjTuGe5M65Z5qxzYiImy/dXExDLnXEt2zNnnRMTGYKQm+VDncY9y51xzTJnnRMTMdl+BbsJhuXOuJbtmbPOiYkMEbuC3bhnuTOuWeasc2IiJtuvYDfBsNwZ17I9c9Y5MZGS7UOdJhiWO+NatmdOVDXoGkJHRFYA89O4iYbAyjS+fzqlu/bWqtoo0QsiMt7ffqyVqtorjfU4YZkrU5hylxGZA8tdOQSSu0xu68rLOicBEJGpqtot6DpSEeXas1nUf29Rrz9bRf33FvX6oywn6AKMMcYYY2JVqHMiIreLyMjKKqaU7ZwnIp+kuG6pNYrIPBE5NvXqjKlctl+ZIFjuTJiU2jkRkQ0x0w4R2RzzfKCrIjPQMAARqSYiz4nIryKyVESuLWkFEXkq7vexVUTWx7xeX0ReE5GNIjJfRM6OW/9sf/5GEXldROqnsi6Qm+q6Fdluhqx7pYhMFREVke2O96thaXpfV8pdf5L7lYjI3SKySETWicgHItKpckoOl4Dac8tdycs+JCKLRWSNiDwhIlUqp+TMUGrnRFVrF03AL8DvY+a9mMyGRCRynwwSkUYilX/JtKoWBf52oD3QGjgauF5EEl7wpKqXxv0+XgZeiVnkcWAb0AQYCDxZ1Mj6/z4NDPJf3wQ8keK631dg3YpsNxPWXQzcDTwFvOhyv4rJXOBS2a+SrP92yrlfAWcCFwBHAPWBScCIZGqLiiDac8tdibm7EegGdAb2AvYH/ppMbRlPVcs1AfOAY+Pm3Q6MAV4A1gMzgW5x69wAfA1sxfvocnfgM2At8BXQI2b584A5/nvNBQbGzP8EeBBY47/WO2a95sCbwGpgNnBxXI0jY54Pwrs6fRVwc6KfK2bZG/zX7wDalPf/Kon/08XA8THP7wJGlWO9Wv7/0VExz7cBe8UsMwIY6j++F3gp5rV2/vJ1bF0368b9/u4Gnrf9Kvj9yq9lTMzzTsCWyq4pbJPlLvDcTQXOjHl+NrAg6FyEaaqMC2JPBkYB9fxAPRb3+lnASf7rTYBxeI1zfeA64D9+j7YW8CheSOsAhwIzYt7nYOBHvI9X3Q/8K6YXPApYiBfqM4B7RaRnfKEi0hF4Ei/QzYEGQIuSfjBV/RswAGgMTBWR90VkkIjUjHvfG0VkbUlTzHK9RORHEZktIncAzfB26CJf4TWOZTkdWAF85D/fCyhQ1VklvFen2O2o6s/4f2TLu64/XLkceCPZdSuy3QxaN1mVuV897q+zDHiJaO1XW/1TEIUl7Vdx6+xOcvvVKKCdiOzlD6sPBsaXVHsWqKzc9QX+BeQC9xC99jzduQOQuMctRGS3UpbPKpXROflEVd9W1UK8o8guca8/qqoLVHUzcA7wtr/8DlV9D68HeaK/7A6gs4jUUNUlqjoz5n3mq+oz/naG4wWhiYi0BA4DblDVLao6A3gWODdBrWcAb6nqR6q6FbjF32aJVPVzVb0ML/xP4u2cC0Xk2ZhlhqpqvZImABHJxfsj0RvoiDecDLAuZnPr8I7QyzIYeEH9LjdQG/g1bpnY96odt53Y18u77vNArxTXrch2M2XdZFXWfpUDVAcewBs67gPE3j8g1PsVcBze8Pf3ifarBGr7/5Z3v1qCdxT/I7AZb7+8prTaM1xl5O4k4CFgI3A93qjA7lFqz0l/7sYDV/sduabAVf78miUsn3Uqo3OyNObxJqB63PnIBTGPWwNnxvVCDweaqepGoD9wKbBERMaJSIdE21HVTf7D2nghW62q62OWnQ/kJ6i1eWw9/jZXleeH9MP/NV7vfxveucJkHATMVtU5qrqN364XqRuzTF28IdASiUgroAfe0GuRDXHvE/9epb1ernVV9SO8Ydak163IdjNo3WRVyn6Fl9MvgdPwrjOoBlyUaDth3K/iclceG/x/y7tf3QocCLTE68TdAfwv/mg6i1RG7g4CfsLr6F2Edx3Gq1Fqzx3k7h68/XIG3mmx14HteKObBjf3OYk9SlsAjIjridbye6qo6gRVPQ6vUf0BeKYc778YqC8isT3UVsCiBMsuwWuEAPAboAalvbmINBDvUxaTgf/hDVMerardY5b5ixS/Er7Y5C+WT/Edezbezh97ZNIF7zxvaQYBn6rqnJh5s4A8EWlfwnvNjN2OiLTF+yM1K9l1gSqprluR7WbAupWtvPtVPjAjZr+agze0XZbQ7Fd4/6/7lLBfFaOqa/x6yrtfdQVGq+pCVS1Q1eeB3fFGN82uyswd3u9zQUx7fgXeqFSk2nPSmDtV3ayqV6pqvqq2xetUTVPVUkd+sopW/AKq2IuT9sALb16idfCCtBQ4AS8U1fFGAVrgnb/si3fRYQ7eEcyHGnMBVdy2FdjTf/wx3rnR6sB+eL3PY+NrxDv/twGvd18V74KsgvifK2YbF+L1fF/BG6rMLe//V4L3OgN4Nub5IGAa8CFeY9gBL9y9ynifH4ELEswfhfcJnlp4w6LrgE4xP/eveJ9IqAWMJOZCrSTW3QfvwrdU1q3IdjNl3Ty8jN6HN2RenfTvVxcC7/LbfvUasChK+5X/83+bxL42lHLuV8BteKd1mvj/P4PwTkfUS3Vfj8KU5twN8bNf1J6fC0wmYu15mnOXjzfyI3gXFS8g5mJam9Rt58Sfd7D/C1yNd1HnOLyecTN//jq8P4AfAB3LGeYWwFv+e/4MXFpKjYPxhrfLc3V3R6B+pfxHwyHAhJjnN+F9dOw5vD9oy4BrY15v5e94reLeYyNxn/7wX6uPNzS40f/5zo57/Wx//ka8i1rrp7DuJr/WVNatyHYzZd3b8XIbO62Nzx+Vu1/18ecV7VdzgIeitF+R/B+JapRzv8L7A/g43h+SX4HplHGAkAlTCRmqrNydArzPb+35Zj93kWrP05y7I/1aN+EdcA4MOhNhm+y7dRzxz9vOAo7BG6KcgveHrazTOKEhInvgXYCW7PU2JiCWO+NaJmQOLHdBc3HNiQFUtQC4EpiAdyOzMVHaWUXkZbwbVO0tIgtF5MKgazJls9wZ16KeObDchYGNnBhjjDEmVGzkxBhjjDGhYp0TY4wxxoSKdU6MMcYYEyqR+6ZgFySvhkrVVO44Hg6/26dV0CWkbP78eaxcuTLhN4fm1m2tWrC52DzdvGKCqpb0zZ+RYZkL1vTp01aqaqNEr8XnLlMyB5a7oJWUu0xu68rLOicJSNU6VNu7X9BlpOzTL+K/qys6Dju4W4mvaeEWqnUcWGzelmkPN0x3TS5Y5oJVo4rML+m1+NxlSubAche0knKXyW1deVnnxESIQE5u0EWYrGO5M65Z5qxzYqJDBHItssYxy51xzTJnnRMTJQK5VYIuwmQdy51xzTJnnRMTHULWD3WaAFjujGuWOeucmAgRgbzsPpowAbDcGdcsc9Y5MVFi52FNECx3xjXLXHb/9CZaxK5gNwGw3BnXLHPWOTFRYheJmSBY7oxrljnrnJjoENthTQAsd8Y1y5x1TkzEZPlQpwmI5c64luWZs86JiQ67MZEJguXOuGaZs86JiQ7vGjH7Im3jluXOuGaZs86JiRQhJye7d1gTBMudcc0yZ50TEx12NGGCYLkzrlnmyO6fPg2eum0g8yfex9RX/rLLa1cP6snmLx+jQb1aANSrU4PRf7+YyaNv4uMR19GxXTPX5Sbl3Qnj2a/T3nTqsCcP3D/U+fYFITc3t9hkPJa79InPnfFY5tLH2jrrnFS6EWM/p+8Vj+8yv0WTehzTfR9+WbJ657zrLzyBr35cyEH97+PCW0bw4J/PcFlqUgoLC/njVVfwxth3+PLr73hl1Mt8/913bosQkBwpNhmP5S6N4nJnPJa5NLK2zjonle3T6T+zet2mXebff93p3PzI66jqznkd2jblwymzAJg1bxmtm9encf06zmpNxpTJk2nXbk/atG1L1apVObP/AN4a+4bTGuxoomSWu/SxkZPELHPpY22ddU6c6NNjXxYvX8s3sxYVm//NrEX07dkFgG6dWtOqWX3ym9QLoMKyLV68iBYtWu58np/fgkWLFpWyRhr452FjJ1Myy10licudKZllrpJYW5cdnRMR6SUiP4rIbBG50eW2a1SvwvUXnMCdT47b5bUH/+89dqtTk89H3chlA47iqx8XUli4w2V5kSL+FeyxU1gFmTmw3FWm+NyFmbV1mSFKbV26ZPyndUQkF3gcOA5YCEwRkTdV1clJxLYtGtE6vwGTR98EQH7jekx66QaOGPQAy1at55LbR+5c9odxdzB30SoXZSWtefN8Fi5csPP5okULyc/Pd1tERK5gDzpzYLmrVJa7crHMVaKIZC6dMr5zAhwEzFbVOQAiMgroCzjZYWfOXkzrY27a+fyHcXdw2MD7WbV2I7vVrsGmLdvYXlDI+aceyifTZ7N+4xYXZSWt24EHMnv2T8ybO5fm+fm8MnoUz494yXkdEdlhA80cWO4qm+WubJa5yhWRzKVNNnRO8oEFMc8XAgena2PD7zuPIw5oT8N6tZk9/i7ueupthr8+KeGyHdo25Zk7B6GqfP/zEi6948V0lVVheXl5PPTIY/z+pBMoLCxk8HkX0LFTJ6c1iETmxkROMweWu3Sy3CVmmUufCGUubbKhc1IuIjIEGAJAldopv8/gm54v9fUOJ9228/EXX89lv1PuTHlbrvXqfSK9ep8YaA2ZdDRRWZkDy126We52ZZlLr0zKXCqyoXOyCGgZ87yFP68YVR0GDAPIqdlY4183wRMRcqOxw1rmMojlzrgWocylTeg7JyLyT6DEHUhVryrjLaYA7UWkDd6OOgA4u/IqNC7lOLoZUQVzZ5nLMC5yZ22dieWqrQur0HdOgKkVWVlVC0TkSmACkAs8p6ozK6Uy45QI5OU522FTzp1lLrM4zJ21dQZw3taFUug7J6o6PPa5iNRU1V1vS1j6e7wNvF2phRnnREh6qFNEngP6AMtVtbM/rz4wGtgDmAf0U9U1IiLAI8CJwCbgPFWd7q+TVO4sc5kjqNxZW5e9gmzrwiIyJ7VE5BAR+Q74wX/eRUSeCLgs45SQk1N8KofngV5x824EJqpqe2Ci/xygN9Den4YAT1ruTHzuyul5Us/dCMtctnPf1lVK2ZUoMp0T4GHgBGAVgKp+BRwZZEHGraKjidipLKr6EbA6bnZfoGhEbjhwSsz8F9TzOVAP76ZWlrssFp+78qhg7toCg7DMZa0g2joRCdVXRYf+tE4sVV3gjUbtVBhULSYYubm7HEE0FJHYc/XD/E8jlKaJqi7xHy8FmviPE90noonlzsTlLpXMQflztw2ID7plLssE0NblA0sIiSh1ThaIyKGAikgV4Grg+4BrMg6JJLyCfaWqdkv1PVVVRaS0j1Mus9xltwS5q1DmoMzcbQX2wzKXtQJq60IlSqd1LgWuwOvdLQa6+s9N1pCkhzpLsKxoCNP/d7k/P9F9Iq7CcpflpDIyB+XP3a/ASVjmslggbZ3jr14uXWQ6J6q6UlUHqmoTVW2kqueoaji/OcqkRSrnYUvwJjDYfzwYeCNm/rni6Q6sU9WZlrvslso1JyUob+5Wq+rplrnsFVBbF5pTOhChzomItBWRsSKyQkSWi8gbItI26LqMW8lewS4iLwOTgL1FZKGIXAgMBY4TkZ+AY/3n4H0Ecw4wG3gGuNxyZ4CkP61TwdzdY5kzrtu6dPwMFRGla05ewvvkxKn+8wHAy6T5C9VMeHi3dE7uxkSqelYJLx2TYFklbvhcRD7HcpfVXOfOMmeCaOvCJjIjJ0BNVR2hqgX+NBKoHnRRxh0RyMvNKTY5YLnLcvG5c8Ayl+UCautCJfQjJ/4d7gDeEZEbgVF43z/RH7sTYtbJdffdOpY7s5OL3FnmTCxXbV1Yhb5zAkzD20GLflOXxLymwE3OKzKBECDP3Q5ruTOA09xZ5gzgvK0LpdB3TlS1TdA1mHAQEWfDm5Y7U8RV7ixzpojLti6sQt85iSUinYGOxJx/VdUXgqvIuBbE14hb7ozr3FnmTBBtXZhEpnMiIrcBPfB22LfxvrjoE8B22Cwh4n6o03JnXOfOMmeCaOvCJkrjRmfgfSRqqaqeD3QBdgu2JONS0VCn4yvYLXdZLj53DljmslxAbV2oRGbkBNisqjtEpEBE6uLdhrdlWSuZzBLAUKflzrjOnWXO2GmdoAtIwlQRqYd3N7tpwAa8u+GZLBHQFeyWuywXQO4sc1nOPq0Toc6JqhbdXvcpERkP1FXVr4OsybhVdGMilyx3xnXuLHMmiLYubELfORGR/Ut7TVWnu6zHBEcQlzdhs9wZwF3uLHOmiMu2LqxC3zkB/l7Kawr0rOwN7tO+BWPeGlr2giHVcsjooEtI2dr5a0p+0e0V7E5zt3e7fJ7/992V+ZZOdbohg29g6i53ztu6/JZNuObhayv7bZ0554VpQZeQHvZpnfB3TlT16KBrMOEgQJUkvwwrVZY7U8RV7ixzpojLti6sQt85MaaIiH3fhHHPcmdcs8xZ58REiABVsnyHNe5Z7oxrljnrnJgI8W5MlN07rHHPcmdcs8xF6A6x4jlHRG71n7cSkYOCrsu4U/TZ/9gp7du03GW9+NylfXuWuawXRFsXNpHpnABPAIcAZ/nP1wOPB1eOcU4C2WEtd9lO3HZOsMyZYNq6UInSaZ2DVXV/EfkSQFXXiEjVoIsy7gR0BbvlLssFkDvLXJazT+tEq3OyXURy8T7vj4g0AnYEW5JxybuC3flmLXdZLoDcWeayXEBtXahE6cd/FHgNaCwi9+B9hfi9wZZkXCo6moidHLDcZbn43DlgmctyAbV1oRKZkRNVfVFEpuF9lbgAp6jq9wGXZRwSEecfr7PcGde5s8yZINq6sIlM50REWgGbgLGx81T1l+CqMi4J7m9MZLkzrnNnmTNBtHVhE5nOCTAO7xysANWBNsCPQKcgizLupHpjIhGZh/eJh0KgQFW7iUh9YDSwBzAP6Keqib7Yx3KX5VLJnWXOVERAbV2oROaaE1XdV1X38/9tDxwETAq6LuOOCOTl5BSbknC0qnZV1W7+8xuBiX6WJvrPd2G5M/G5S4JlzqQkiLYubCLTOYnnf334wUHXYdzKkeJTBfQFhvuPhwOnlGcly112sswZ14Ju64IWmdM6IhL7vd45wP7A4oDKMQEQJNkjiCIKvCsiCjytqsOAJqq6xH99KdAk4TYtd1kvxdxZ5kzKgmjrwiYynROgTszjArzzsv8JqJZyWbJ4IX+5egirVi5HRDjj7PMZdNHlALz43FOMGj6MnNxcjux5An/6690BV+t55PwDOa5Lc1b+upUjbx0PwG1nduGErs3ZVrCDeSs2cNW/JvPr5u3sXqsqz11+KL9rU59Rn87jxhenp7U2EciTXQ4hGorI1Jjnw/wdMtbhqrpIRBoD74nID7Evqqr6O3Mikcvd1q1buOysk9i2bSuFBYX07HUyF//xJlSVp/5xN/975w1ycnM57ewL6D/4kqDLBWBo/33puU9jVm3YRu8HPwag935NueqE9uzZuDanPfIZ3yxcB0C9mlV4fPD+7NtyN/4zZSF3vPZdWmtLkDvLXAKb1//KmAduYsncWYgI/W8YyjcfTWDmZ/8jr0oVGjRvxYAb7qdGnbpBl5pQn06NOWavhijwy5rNPP7xPIYc2oqOTeuwaVshAI9/PI95qzenvZaA2rpQiUTnxL8hUR1VvS7oWpKRl5vHn2+9l477dmXjhvX0630Ehx7Zk1UrlvP+u+P4z7uTqFqtGqtWrgi61J1GfTqPf02czWMX/TaK/OF3S7n7P19TuEO55Yz9uPqkfbjr31+zdXshQ1//lg75u7FP/m5O6svZdYddGXNuNSFVXeT/u1xEXsM7h79MRJqp6hIRaQYsj18vqrmrWrUaj414g5q1alOwfTtDBvTmkKOOZd7Ps1i+ZBGj351MTk4Oq1eFJ3f/mbKQEZ/M58GzuuycN2vpei5/fjp3n9G52LJbC3bwj/Gz2KtpHfZqWttJfXG5s8wl8Ppjd7L3QUcy+M7HKdi+je1btrC120ZOvPjP5Obl8dbTf2PiS0/S55Ibgi51F/VrVqF3x8Zc8+pMthUq1x7dhsPa1AdgxJSFfD5vrfOaXLZ1YRT6a05EJE9VC4HDgq4lWY2aNKXjvl0BqFW7Dm3b782ypYsZPeJZLrziWqpWqwZAg4aNAqyyuEmzVrBm49Zi8z6YuYzCHV5ne9qcVTTfvSYAm7YV8sVPK9m6vdBJbal8GZaI1BKROkWPgeOBb4E3gcH+YoOBN+LWi2zuRISatbw/2gUF2ynYvh1EePWl57jgyuvJ8YeL6zcIT+6mzFnD2k3bi837eflG5q7YuMuym7cVMm3uGrYVBJO7MpfPwsxt3rCeOV9N4eCT+gGQV6UqNerUZe8DjyA3zzsGbt2xK2tXLA2yzFLlilA1N4ccgWq5OazZtC2wWly2dWEVhZGTyXjnXGeIyJvAK8DOFktVXw2qsGQsWjCf77/9mv1+142/3/1Xpn3xGY/+7U6qVavGn265h327HhB0ieVy9uFteH3ygkC2LQh5knR/ugnwmnhHIXnAS6o6XkSmAGNE5EJgPtAvbr1I566wsJDzTunBwvlzOf2cC+nctRsLf5nLf99+lQ/fHUe9+g249ta/0WqPdkGXGnop5C7rMrd6yQJq1avPqKHXs/jnH2ixV2dO+cMtVKtRc+cyk9/+N12PPinAKku2etN23vx2GU/235dtBTv4evGvfLV4PYe3q89ZB+RzZtdmfLN4PSOnLqJgR/rPijhu60IpCp2TItWBVUBPfrsHgAKl7rAi8hzQB1iuqp1LWzZdNm3cwDVDzuGG24dSu05dCgsL+HXtGl4a+z++nTGN6y4bzPjPvkF2HcYLlWv67EPBDuXfn88PpgCBZK8RU9U5QJcE81fh3YGzLJHMXW5uLiPGfsz6X9dxw2Xn8POs79i+bRtVq1bn+dff5/0JY7nnxit5etQ7rkuLniRzl42Z21FYwKJZMzn1qtto3bErr//zTv730lP0vtC7tve/Ix4nJzeX/Y/r67KscqtVNZcDW+3GFa98y8atBfypZzuOaFefF6cuYu3mAvJyhEsPa80p+zXl3zOWlP2GFRVMWxcqoT+tg/f9EtfiDU994/870//323Ks/zzQK23VlWH79u38ccg5nHRqP4470dsxmzTN59jeJyMi7Pu7bkhODmtWrwyqxHIZcNgeHLdfcy4b9nlgNQiQJznFpjSKdO6K1Km7Gwd0P4LPP5pI46bNOfqE3wPQ4/g+zP5hZsDVRUN87tIospnbrVEzdmvUlNYduwKw31G9WfSTl6/J7/yb7ya9z8C/PhTaA7D9mtdh+YZt/LqlgEKFL+avYe/GtVi7uQCAgh3K+z+tpH3DmmW8U+Vw3NaFUhR+4lygtj/ViXlcNJVKVT8CVqezwFK2za3XXUHbPfdm8JA/7Jzfs1cfJn/2EQDz5vzE9m3b2L1+wyBKLJeenZtyZe8ODPrnJ2ze5uY8fyKpnIetgMjmbs2qlaz/1ftky5Ytm5n86fu0btueI489kWmfe5+Emf7Fp7Rqs2cQ5UVOstecVEBkM1e3QSPqNW7G8l/mAPDTtM9o0npPfvjiQz4Y9QwX3Ps0VavXCKK0clm5cRt7NapFVf8L9vZtVpdFa7dQr8ZvJxcObF2PX9ZucVKP47YulKJwWmeJqt4ZdBGp+HLKJMb+52Xad+jE6ccfCsDVN9zGaf0H8dc/Xc4pxxxElSpVuffhp0NzRPH0Jd05bO/G1K9dja8e/D33v/EtV5+4D1Wr5PLvPx0FwNSfV/HnEdMAmHZ/H+pUz6NqXg69f5fPmf/4kFmLf01PcSKJrmBPl8jmbuWKpdz158sp3FGI7tjBMSeeyuE9e9Gl2yHcdu3FjPq/J6hRszZ/ufeRoEvd6eFzunJwu/rsXqsqn9xyNI9M+Il1m7Zz66kdqV+7Ks9e1I3vFv/K+cOmAPDhzT2oXT2PKrk5HNe5CecNm8LsZRvSU5y73EU2cwCnXnUbL959DYUF26nfrCUDbryfhy85hYLt23j6T971mK07duWMP4XjtgmxflqxiUnz1vBA344UqjJ31Sbe+3ElNx+/J3WrV0EE5q3axLDPHH29kdu2LpSi0Dlx8hsSkSHAEIBm+S0r5T33P+hQvl24PuFrf/vns5Wyjcp2ydO7nrZ58eO5JS5/wPVvpbOcYryhTmc7bNo3FJu5ps1bVNr7tu/QmRfGfrTL/Dp1d+Mfz46ptO1Upj+OnJFw/rvfLks4/6h7PkhfMXEc5s55W7d7k+aV9r757TtyzbDiHwT5y0vvV9r7p9uYL5cw5svi15PcMf6nQGpx3NaFUhRO6zi5kEdVh6lqN1XttnuD8J5iyXY5OVJsSqO05y42c/VCfFrPkDGZg+K5q7VbfRebNClw2NaFUuhHTlQ1kHOoJpxcHUxY7kwsF7mzzJlYWT5wEomRkwoRkZfxvtFzbxFZ6H/W20SQiHejpNgprCx3mSM+d2FlmcscUWrr0iX0IycVpapnBV2DqSzRGd603GWSaOTOMpdJopG5dMr4zonJHIINdRr3LHfGNcucdU5MxGTj8KYJnuXOuJbtmbPOiYkMEbJ+qNO4Z7kzrlnmrHNiIibLDyZMQCx3xrVsz5x1TkykZPtQpwmG5c64lu2Zs86JiQwb6jRBsNwZ1yxz1jkxEZPlBxMmIJY741q2Z846JyYyhOy8GZEJluXOuGaZs86JiZhsH+o0wbDcGdeyPXPWOTHRIZDl+6sJguXOuGaZs86JiQ4BcrJ8qNO4Z7kzrlnmrHNiIibbhzpNMCx3xrVsz5x1TkxkiA11mgBY7oxrljnrnJiIyfahThMMy51xLdszZ50TExmCkJvthxPGOcudcc0yBzlBF2BMMkSKT2UvL71E5EcRmS0iN6a/QpOJksmct7zlzlRMsm2dt07m5M46JyYyRLzvm4idSl9ecoHHgd5AR+AsEenooFSTQeJzV/byljtTMcm2dd46mZU765yYSMnJkWJTGQ4CZqvqHFXdBowC+qa9SJNxksgcWO5MJUiyrYMMy511TkykJDnUmQ8siHm+0J9nTFKSHF633JkKS+G0Tkblzi6ITeC7r79c2blFnflp3ERDYGUa3z+d0l1765Je+HL6tAm1quY0jJtdXUSmxjwfpqrD0lNa+vzw7YyV3ffc3TJXsjDlLiMyB7Bw1rcr/9SjneWuZIHkLpPbuvKyzkkCqtoone8vIlNVtVs6t5EuQdauqr2SXGUR0DLmeQt/XuhY5kpnuUsPy13pgqo/hcxBhHJXHnZax2SyKUB7EWkjIlWBAcCbAddkMp/lzgQho3JnIycmY6lqgYhcCUwAcoHnVHVmwGWZDGe5M0HItNxZ5yQYUT5PGKnaVfVt4O2g6wiBSP3eEohU/Za7nSL1e0sgUvVnUu5EVYOuwRhjjDFmJ7vmxBhjjDGhYp0Th6J8a2EReU5ElovIt0HXYpJjuTOuRTlzYLkLA+ucOJIBtxZ+Hkjl420mQJY741oGZA4sd4Gzzok7kb61sKp+BKwOug6TNMudcS3SmQPLXRhY58SdjLq1sIkMy51xzTJnKsw6J8YYY4wJFeucuJNRtxY2kWG5M65Z5kyFWefEnYy6tbCJDMudcc0yZyrMOieOqGoBUHRr4e+BMVG6tbCIvAxMAvYWkYUicmHQNZmyWe6Ma1HPHFjuwsDuEGuMMcaYULGRE2OMMcaEinVOjDHGGBMq1jkxxhhjTKhY58QYY4wxoWKdE2OMMcaEinVOKkhECkVkhoh8KyKviEjNCrzX8yJyhv/42dK+LEtEeojIoSlsY56INCzv/LhlNiS5rdtF5LpkazRls9yVurzlLg0sc6Uub5mrZNY5qbjNqtpVVTsD24BLY18UkbxU3lRVL1LV70pZpAeQ9A5rMoblzrhmmTPOWOekcn0M7On39D8WkTeB70QkV0QeEJEpIvK1iFwCIJ7HRORHEfkv0LjojUTkAxHp5j/uJSLTReQrEZkoInvgNQzX+EcyR4hIIxH5j7+NKSJymL9uAxF5V0RmisizgJT1Q4jI6yIyzV9nSNxrD/nzJ4pII39eOxEZ76/zsYh0qJT/TVNeljvLnWuWOctceqmqTRWYgA3+v3nAG8BleD39jUAb/7UhwF/9x9WAqUAb4DTgPSAXaA6sBc7wl/sA6AY0wvuGz6L3qu//eztwXUwdLwGH+49bAd/7jx8FbvUfnwQo0DDBzzGvaH7MNmoA3wIN/OcKDPQf3wo85j+eCLT3Hx8M/C9RjTZZ7ix30Z0sc5Y5l1NKw3CmmBoiMsN//DHwL7whyMmqOteffzywn/jnWIHdgPbAkcDLqloILBaR/yV4/+7AR0XvpaqrS6jjWKCjyM6DhboiUtvfxmn+uuNEZE05fqarRORU/3FLv9ZVwA5gtD9/JPCqv41DgVditl2tHNswFWO5s9y5ZpmzzDljnZOK26yqXWNn+MHdGDsL+IOqTohb7sRKrCMH6K6qWxLUUm4i0gNv5z9EVTeJyAdA9RIWV3+7a+P/D0zaWe4sd65Z5ixzztg1J25MAC4TkSoAIrKXiNQCPgL6++dpmwFHJ1j3c+BIEWnjr1vfn78eqBOz3LvAH4qeiEhX/+FHwNn+vN7A7mXUuhuwxt9ZO+AdzRTJAYqOiM4GPlHVX4G5InKmvw0RkS5lbMO4YbkzrlnmTKWwzokbzwLfAdNF5FvgabxRq9eAn/zXXsD7FsxiVHUF3nncV0XkK34bahwLnFp0kRhwFdBNvIvQvuO3K+nvwNvhZ+INef5SRq3jgTwR+R4YitdgFNkIHOT/DD2BO/35A4EL/fpmAn3L8X9i0s9yZ1yzzJlKYd9KbIwxxphQsZETY4wxxoSKdU6MMcYYEyrWOTHGGGNMqFjnxBhjjDGhYp0TY4wxxoSKdU6MMcYYEyrWOTHGGGNMqFjnxBhjjDGh8v8zpsP9/VMLlQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x432 with 18 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lr = LogisticRegression(C = 0.01, penalty='l1',solver='liblinear')\n",
    "lr.fit(X_train_under_sample, Y_train_under_sample.values.ravel())\n",
    "Y_pred_unser_sample_pro  = lr.predict_proba(X_test_under_sample.values) # 预测一个概率值\n",
    "\n",
    "thresholds = np.arange(0.1,1,0.1) # [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]\n",
    "plt.figure(figsize=(8,6))\n",
    "j = 1\n",
    "for i in thresholds:\n",
    "  y_test_pred_high_recall = Y_pred_unser_sample_pro[:,1] > i\n",
    "  plt.subplot(3,3,j)\n",
    "  j += 1\n",
    "  cnf_matrix = confusion_matrix(Y_test_under_sample,y_test_pred_high_recall)\n",
    "  np.set_printoptions(precision=2)\n",
    "  print('Recall metric in test: ', cnf_matrix[1,1] / (cnf_matrix[1,0]+cnf_matrix[1,1]))\n",
    "  class_names = [0,1]\n",
    "  plot_confusion_matrix(cnf_matrix, labels=class_names, title='Threshold >= %s' % i)\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
